Merged latest V1_6_X branch

4 files changed, 256 insertions, 85 deletions

diff --git a/RepSys/commands/rpmlog.py b/RepSys/commands/rpmlog.py
index 5ba5fdd..3cdcc02 100644
--- a/RepSys/commands/rpmlog.py
+++ b/RepSys/commands/rpmlog.py
@@ -3,10 +3,13 @@
 # This program will convert the output of "svn log" to be suitable
 # for usage in an rpm %changelog session.
 #
-from RepSys import Error
+from RepSys import Error, RepSysTree
 from RepSys.command import *
-from RepSys.log import svn2rpm
+from RepSys.svn import SVN
+from RepSys.log import get_changelog, split_spec_changelog
+from cStringIO import StringIO
 import getopt
+import os
 import sys
 
 HELP = """\
@@ -18,6 +21,9 @@ Options:
     -r REV   Collect logs from given revision to revision 0
     -n NUM   Output only last NUM entries
     -T FILE  %changelog template file to be used
+    -o       Append old package changelog
+    -p       Append changelog found in .spec file
+    -s       Sort changelog entries, even from the old log
     -h       Show this message
 
 Examples:
@@ -30,14 +36,31 @@ def parse_options():
     parser.add_option("-r", dest="revision")
     parser.add_option("-n", dest="size", type="int")
     parser.add_option("-T", "--template", dest="template", type="string")
+    parser.add_option("-o", dest="oldlog", default=False,
+            action="store_true")
+    parser.add_option("-p", dest="usespec", default=False,
+            action="store_true")
+    parser.add_option("-s", dest="sort", default=False,
+            action="store_true")
     opts, args = parser.parse_args()
     if len(args) != 1:
         raise Error, "invalid arguments"
     opts.pkgdirurl = default_parent(args[0])
     return opts
 
-def rpmlog(pkgdirurl, revision, size, template):
-    sys.stdout.write(svn2rpm(pkgdirurl, revision, size, template=template))
+def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort):
+    another = None
+    if usespec:
+        svn = SVN()
+        pkgname = RepSysTree.pkgname(pkgdirurl)
+        #FIXME don't hardcode current/, it may already be in the URL
+        specurl = os.path.join(pkgdirurl, "current/SPECS", pkgname +
+                    ".spec")
+        rawspec = svn.cat(specurl, rev=revision)
+        spec, another = split_spec_changelog(StringIO(rawspec))
+    newlog = get_changelog(pkgdirurl, another=another, rev=revision,
+            size=size, sort=sort, template=template, oldlog=oldlog)
+    sys.stdout.writelines(newlog)
 
 def main():
     do_command(parse_options, rpmlog)
diff --git a/RepSys/commands/submit.py b/RepSys/commands/submit.py
index f1726dc..2869ff4 100644
--- a/RepSys/commands/submit.py
+++ b/RepSys/commands/submit.py
@@ -1,22 +1,18 @@
 #!/usr/bin/python
 from RepSys import Error, config
 from RepSys.command import *
-from RepSys.rpmutil import get_spec, get_submit_info
+from RepSys.rpmutil import get_spec, get_submit_info, svn_url_rev
 from RepSys.util import get_auth, execcmd, get_helper
 import urllib
 import getopt
 import sys
 import re
-
-#try:
-#    import NINZ.client
-#except ImportError:
-#    NINZ = None
+import subprocess
 
 import xmlrpclib
 
 HELP = """\
-Usage: repsys submit [OPTIONS] [URL [REVISION]]
+Usage: repsys submit [OPTIONS] [URL[@REVISION] ...]
 
 Submits the package from URL to the submit host.
 
@@ -28,8 +24,8 @@ The status of the submit can visualized at:
 
 http://kenobi.mandriva.com/bs/output.php
 
-If no URL and revision are specified, the latest changed revision in 
-the package working copy of the current directory will be used.
+If no URL and revision are specified, the latest changed revision in the
+package working copy of the current directory will be used.
 
 Options:
     -t TARGET  Submit given package URL to given target
@@ -44,37 +40,46 @@ Options:
 
 Examples:
     repsys submit
-    repsys submit foo 14800
-    repsys submit https://repos/svn/mdv/cooker/foo 14800
-    repsys submit -r 14800 https://repos/svn/mdv/cooker/foo
+    repsys submit foo
+    repsys submit foo@14800 bar baz@11001
+    repsys submit https://repos/svn/mdv/cooker/foo
     repsys submit -l https://repos
-    repsys submit --define section=main/testing -t 2008.0
+    repsys submit --define section=main/testing -t 2008.1
 """
 
 def parse_options():
     parser = OptionParser(help=HELP)
-    parser.defaults["revision"] = ""
+    parser.defaults["revision"] = None
     parser.add_option("-t", dest="target", default="Cooker")
     parser.add_option("-l", action="callback", callback=list_targets)
     parser.add_option("-r", dest="revision", type="string", nargs=1)
     parser.add_option("-s", dest="submithost", type="string", nargs=1,
             default=None)
-    parser.add_option("--define", action="append")
+    parser.add_option("--define", action="append", default=[])
     opts, args = parser.parse_args()
     if not args:
-        name, rev = get_submit_info(".")
-        args = name, str(rev)
-        print "submitting %s at revision %s..." % args
-    elif len(args) > 2:
-        raise Error, "invalid arguments"
-    opts.pkgdirurl = default_parent(args[0])
+        name, url, rev = get_submit_info(".")
+        args = ["%s@%s" % (url, str(rev))]
+        print "Submitting %s at revision %s" % (name, rev)
+        print "URL: %s" % url
+    if opts.revision is not None:
+        # backwards compatibility with the old -r usage
+        if len(args) == 1:
+            args[0] = args[0] + "@" + opts.revision
+        else:
+            raise Error, "can't use -r REV with more than one package name"
+    del opts.revision
     if len(args) == 2:
-        opts.revision = re.compile(r".*?(\d+).*").sub(r"\1", args[1])
-    elif len(args) == 1 and opts.revision:
-        # accepts -r 3123 http://foo/bar
-        pass
-    elif not opts.list:
-        raise Error, "provide -l or a revision number"
+        # prevent from using the old <name> <rev> syntax
+        try:
+            rev = int(args[1])
+        except ValueError:
+            # ok, it is a package name, let it pass
+            pass
+        else:
+            raise Error, "the format <name> <revision> is deprecated, "\
+                    "use <name>@<revision> instead"
+    opts.urls = [default_parent(nameurl) for nameurl in args]
     return opts
 
 def list_targets(option, opt, val, parser):
@@ -87,9 +92,7 @@ def list_targets(option, opt, val, parser):
     execcmd(command, show=True)
     sys.exit(0)
 
-def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
-    #if not NINZ:
-    #    raise Error, "you must have NINZ installed to use this command"
+def submit(urls, target, define=[], submithost=None):
     if submithost is None:
         submithost = config.get("submit", "host")
         if submithost is None:
@@ -101,13 +104,13 @@ def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
             del type, user, port, path, rest
     # runs a create-srpm in the server through ssh, which will make a
     # copy of the rpm in the export directory
-    if list:
-        raise Error, "unable to list targets from svn+ssh:// URLs"
     createsrpm = get_helper("create-srpm")
-    command = "ssh %s %s '%s' -r %s -t %s" % (
-            submithost, createsrpm, pkgdirurl, revision, target)
-    if define:
-        command += " " + " ".join([ "--define " + x for x in define ])
+    args = ["ssh", submithost, createsrpm, "-t", target]
+    for entry in define:
+        args.append("--define")
+        args.append(entry)
+    args.extend(urls)
+    command = subprocess.list2cmdline(args)
     status, output = execcmd(command)
     if status == 0:
         print "Package submitted!"
@@ -115,7 +118,6 @@ def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
         sys.stderr.write(output)
         sys.exit(status)
 
-
 def main():
     do_command(parse_options, submit)
 
diff --git a/RepSys/log.py b/RepSys/log.py
index 60cf0d5..2eadbe2 100644
--- a/RepSys/log.py
+++ b/RepSys/log.py
@@ -8,6 +8,8 @@ try:
 except ImportError:
     raise Error, "repsys requires the package python-cheetah"
 
+from cStringIO import StringIO
+
 import sys
 import os
 import re
@@ -464,29 +466,68 @@ def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
     data = dump_file(releases[::-1], currentlog=currentlog, template=template)
     return data
 
-
-
-def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
-        submit=False, template=None, macros=[], exported=None):
-    newlines = []
+def _split_changelog(stream):
+    current = None
+    count = 0
+    def finish(entry):
+        lines = entry[2]
+        # strip newlines at the end
+        for i in xrange(len(lines)-1, -1, -1):
+            if lines[i] != "\n":
+                break
+            del lines[i]
+        return entry
+    for line in stream:
+        if line.startswith("*"):
+            if current:
+                yield finish(current)
+            fields = line.split()
+            rawdate = " ".join(fields[:5])
+            try:
+                date = time.strptime(rawdate, "* %a %b %d %Y")
+            except ValueError, e:
+                raise Error, "failed to parse spec changelog: %s" % e
+            curlines = [line]
+            current = (date, count, curlines)
+            # count used to ensure stable sorting when changelog entries
+            # have the same date, otherwise it would also compare the
+            # changelog lines
+            count -= 1
+        elif current:
+            curlines.append(line)
+        else:
+            pass # not good, but ignore
+    if current:
+        yield finish(current)
+
+def sort_changelog(stream):
+    entries = _split_changelog(stream)
+    log = StringIO()
+    for time, count, elines in sorted(entries, reverse=True):
+        log.writelines(elines)
+        log.write("\n")
+    return log
+
+def split_spec_changelog(stream):
+    chlog = StringIO()
+    spec = StringIO()
     found = 0
-    
-    # Strip old changelogs
-    for line in open(specfile):
+    for line in stream:
         if line.startswith("%changelog"):
             found = 1
         elif not found:
-            newlines.append(line)
+            spec.write(line)
+        elif found:
+            chlog.write(line)
         elif line.startswith("%"):
             found = 0
-            newlines.append(line)
-
-    # Create new changelog
-    newlines.append("\n\n%changelog\n")
-    newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
-        template=template, macros=macros, exported=exported))
+            spec.write(line)
+    spec.seek(0)
+    chlog.seek(0)
+    return spec, chlog
 
-    # Merge old changelog, if available
+def get_old_log(pkgdirurl):
+    chlog = StringIO()
     oldurl = config.get("log", "oldurl")
     if oldurl:
         svn = SVN()
@@ -504,20 +545,74 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
                 logfile = os.path.join(tmpdir, "log")
                 if os.path.isfile(logfile):
                     file = open(logfile)
-                    newlines.append("\n")
+                    chlog.write("\n") # TODO needed?
                     log = file.read()
                     log = escape_macros(log)
-                    newlines.append(log)
+                    chlog.write(log)
                     file.close()
         finally:
             if os.path.isdir(tmpdir):
                 shutil.rmtree(tmpdir)
+    chlog.seek(0)
+    return chlog
+
+def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
+        submit=False, sort=False, template=None, macros=[], exported=None,
+        oldlog=False):
+    """Generates the changelog for a given package URL
+
+    @another:   a stream with the contents of a changelog to be merged with
+                the one generated
+    @svn:       enable changelog from svn
+    @rev:       generate the changelog with the changes up to the given
+                revision
+    @size:      the number of revisions to be used (as in svn log --limit)
+    @submit:    defines whether the latest unreleased log entries should have
+                the version parsed from the spec file
+    @sort:      should changelog entries be reparsed and sorted after appending
+                the oldlog?
+    @template:  the path to the cheetah template used to generate the
+                changelog from svn
+    @macros:    a list of tuples containing macros to be defined when
+                parsing the version in the changelog
+    @exported:  the path of a directory containing an already existing
+                checkout of the package, so that the spec file can be
+                parsed from there
+    @oldlog:    if set it will try to append the old changelog file defined
+                in oldurl in repsys.conf
+    """
+    newlog = StringIO()
+    if svn:
+        rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
+                template=template, macros=macros, exported=exported)
+        newlog.write(rawsvnlog)
+    if another:
+        newlog.writelines(another)
+    if oldlog:
+        newlog.writelines(get_old_log(pkgdirurl))
+    if sort:
+        newlog.seek(0)
+        newlog = sort_changelog(newlog)
+    newlog.seek(0)
+    return newlog
 
-    # Write new specfile
-    file = open(specfile, "w")
-    file.write("".join(newlines))
-    file.close()
-
+def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
+        submit=False, sort=False, template=None, macros=[], exported=None):
+    fi = open(specfile)
+    spec, oldchlog = split_spec_changelog(fi)
+    fi.close()
+    another = None
+    if config.getbool("log", "merge-spec", False):
+        another = oldchlog
+    sort = sort or config.getbool("log", "sort", False)
+    chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size,
+                submit=submit, sort=sort, template=template, macros=macros,
+                exported=exported, oldlog=True)
+    fo = open(specfile, "w")
+    fo.writelines(spec)
+    fo.write("\n\n%changelog\n")
+    fo.writelines(chlog)
+    fo.close()
 
 if __name__ == "__main__":
     l = svn2rpm(sys.argv[1])
diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py
index 46fa392..0eeee03 100644
--- a/RepSys/rpmutil.py
+++ b/RepSys/rpmutil.py
@@ -7,6 +7,7 @@ from RepSys.log import specfile_svn2rpm
 from RepSys.util import execcmd
 from RepSys.command import default_parent
 import rpm
+import urlparse
 import tempfile
 import shutil
 import string
@@ -34,6 +35,7 @@ def rpm_macros_defs(macros):
     args = " ".join(defs)
     return args
 
+#FIXME move it to another module
 def rev_touched_url(url, rev):
     svn = SVN()
     info = svn.info2(url)
@@ -50,6 +52,51 @@ def rev_touched_url(url, rev):
             touched = True
     return touched
 
+def svn_url_rev(url, retrieve=True):
+    """Get the revision from a given URL
+
+    If the URL contains an explicit revision number (URL@REV), just use it
+    without even checking if the revision really exists.
+
+    The parameter retrieve defines whether it must ask the SVN server for
+    the revision number or not when it is not found in the URL.
+    """
+    parsed = urlparse.urlparse(url)
+    path = os.path.normpath(parsed[2])
+    dirs = path.rsplit("/", 1)
+    lastname = dirs[-1]
+    index = lastname.rfind("@")
+    rev = None
+    if index != -1:
+        rawrev = lastname[index+1:]
+        if rawrev:
+            try:
+                rev = int(rawrev)
+                if rev < 0:
+                    raise ValueError
+            except ValueError:
+                raise Error, "invalid revision specification on URL: %s" % url
+    if rev is None and retrieve:
+        # if no revspec was found, ask the server
+        svn = SVN()
+        rev = svn.revision(url)
+    return rev
+
+def strip_url_rev(url):
+    """Removes the @REV from a string in the URL@REV scheme"""
+    parsed = list(urlparse.urlparse(url))
+    path = os.path.normpath(parsed[2])
+    dirs = path.rsplit("/", 1)
+    name = lastname = dirs[-1]
+    try:
+        index = lastname.rindex("@")
+    except ValueError:
+        pass
+    else:
+        name = lastname[:index]
+    parsed[2] = os.path.join(dirs[0], name)
+    return urlparse.urlunparse(parsed)
+
 def get_srpm(pkgdirurl,
              mode = "current",
              targetdirs = None,
@@ -83,6 +130,7 @@ def get_srpm(pkgdirurl,
         elif mode == "pristine":
             geturl = os.path.join(pkgdirurl, "pristine")
         elif mode == "current" or mode == "revision":
+            #FIXME we should handle revisions specified using @REV
             geturl = os.path.join(pkgdirurl, "current")
         else:
             raise Error, "unsupported get_srpm mode: %s" % mode
@@ -106,9 +154,6 @@ def get_srpm(pkgdirurl,
             submit = not not revision
             specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
                     template=template, macros=macros, exported=tmpdir)
-        #FIXME revisioreal not needed if revision is None
-        #FIXME use geturl instead of pkgdirurl
-        revisionreal = svn.revision(pkgdirurl)
         for script in scripts:
             #FIXME revision can be "None"
             status, output = execcmd(script, tmpdir, spec, str(revision),
@@ -123,25 +168,30 @@ def get_srpm(pkgdirurl,
             (topdir, builddir, rpmdir, sourcedir, specdir, 
              srcrpmdir, patchdir, packager, spec, defs))
 
-        if revision and revisionreal:
-            #FIXME duplicate glob line
-            srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
-            srpminfo = SRPM(srpm)
-            release = srpminfo.release
-            srpmbase = os.path.basename(srpm)
-            os.rename(srpm, "%s/@%s:%s" % (srpmsdir, revisionreal, srpmbase))
-        srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
+        # copy the generated SRPMs to their target locations
+        targetsrpms = []
+        urlrev = None
+        if revname:
+            urlrev = revision or svn_url_rev(geturl)
         if not targetdirs:
             targetdirs = (".",)
-        targetsrpms = []
-        for targetdir in targetdirs:
-            targetsrpm = os.path.join(os.path.realpath(targetdir), 
-                    os.path.basename(srpm))
-            targetsrpms.append(targetsrpm)
-            if verbose:
-                sys.stderr.write("Wrote: %s\n" %  targetsrpm)
-            execcmd("cp -f", srpm, targetdir)
-        os.unlink(srpm)
+        srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
+        if not srpms:
+            # something fishy happened
+            raise Error, "no SRPMS were found at %s" % srpmsdir
+        for srpm in srpms:
+            name = os.path.basename(srpm)
+            if revname:
+                name = "@%s:%s" % (urlrev, name)
+            for targetdir in targetdirs:
+                newpath = os.path.join(targetdir, name)
+                targetsrpms.append(newpath)
+                if os.path.exists(newpath):
+                    # should we warn?
+                    os.unlink(newpath)
+                shutil.copy(srpm, newpath)
+                if verbose:
+                    sys.stderr.write("Wrote: %s\n" %  newpath)
         return targetsrpms
     finally:
         if os.path.isdir(tmpdir):
@@ -668,6 +718,7 @@ def get_submit_info(path):
     if len(toks) < 2 or toks[-1] != "current":
         raise Error, "unexpected URL received from 'svn info'"
     name = toks[-2]
+    url = "/".join(toks[:-1])
 
     # Finally, guess revision.
     max = -1
@@ -686,6 +737,6 @@ def get_submit_info(path):
     if max == -1:
         raise Error, "revision tag not found in 'svn info' output"
     
-    return name, max
+    return name, url, max
 
 # vim:et:ts=4:sw=4