Allow submitting packages at once

It adds a new syntax for repsys submit, in which each package version can
be specified using the NAME@REV format.

It also required to change create-srpm to accept the new format. It should
still be compatible with older clients.

5 files changed, 155 insertions, 101 deletions

diff --git a/CHANGES b/CHANGES
index 43d0174..66b81ae 100644
--- a/CHANGES
+++ b/CHANGES
@@ -14,6 +14,8 @@
   log section
 - added rpmlog options: -o to append the old changelog, -p to append the
   changelog found in the spec, and -s to resort all changelog entries
+- allow submitting many packages at once
+- package revisions in submit are now specified with name@nnn
 - fixed bad url used when using -v in getsrpm
 - make 'repsys submit' without package name or revision number work again
 - if REPSYS_CONF is set, /etc/repsys.conf and ~/.repsys/config will not be
diff --git a/RepSys/commands/submit.py b/RepSys/commands/submit.py
index f2f1cdb..2869ff4 100644
--- a/RepSys/commands/submit.py
+++ b/RepSys/commands/submit.py
@@ -1,22 +1,18 @@
 #!/usr/bin/python
 from RepSys import Error, config
 from RepSys.command import *
-from RepSys.rpmutil import get_spec, get_submit_info
+from RepSys.rpmutil import get_spec, get_submit_info, svn_url_rev
 from RepSys.util import get_auth, execcmd, get_helper
 import urllib
 import getopt
 import sys
 import re
-
-#try:
-#    import NINZ.client
-#except ImportError:
-#    NINZ = None
+import subprocess
 
 import xmlrpclib
 
 HELP = """\
-Usage: repsys submit [OPTIONS] [URL [REVISION]]
+Usage: repsys submit [OPTIONS] [URL[@REVISION] ...]
 
 Submits the package from URL to the submit host.
 
@@ -28,8 +24,8 @@ The status of the submit can visualized at:
 
 http://kenobi.mandriva.com/bs/output.php
 
-If no URL and revision are specified, the latest changed revision in 
-the package working copy of the current directory will be used.
+If no URL and revision are specified, the latest changed revision in the
+package working copy of the current directory will be used.
 
 Options:
     -t TARGET  Submit given package URL to given target
@@ -44,39 +40,46 @@ Options:
 
 Examples:
     repsys submit
-    repsys submit foo 14800
-    repsys submit https://repos/svn/mdv/cooker/foo 14800
-    repsys submit -r 14800 https://repos/svn/mdv/cooker/foo
+    repsys submit foo
+    repsys submit foo@14800 bar baz@11001
+    repsys submit https://repos/svn/mdv/cooker/foo
     repsys submit -l https://repos
-    repsys submit --define section=main/testing -t 2008.0
+    repsys submit --define section=main/testing -t 2008.1
 """
 
 def parse_options():
     parser = OptionParser(help=HELP)
-    parser.defaults["revision"] = ""
+    parser.defaults["revision"] = None
     parser.add_option("-t", dest="target", default="Cooker")
     parser.add_option("-l", action="callback", callback=list_targets)
     parser.add_option("-r", dest="revision", type="string", nargs=1)
     parser.add_option("-s", dest="submithost", type="string", nargs=1,
             default=None)
-    parser.add_option("--define", action="append")
+    parser.add_option("--define", action="append", default=[])
     opts, args = parser.parse_args()
     if not args:
         name, url, rev = get_submit_info(".")
-        args = url, str(rev)
-        #FIXME bad place for output
+        args = ["%s@%s" % (url, str(rev))]
         print "Submitting %s at revision %s" % (name, rev)
-        print "URL: " + url
-    elif len(args) > 2:
-        raise Error, "invalid arguments"
-    opts.pkgdirurl = default_parent(args[0])
+        print "URL: %s" % url
+    if opts.revision is not None:
+        # backwards compatibility with the old -r usage
+        if len(args) == 1:
+            args[0] = args[0] + "@" + opts.revision
+        else:
+            raise Error, "can't use -r REV with more than one package name"
+    del opts.revision
     if len(args) == 2:
-        opts.revision = re.compile(r".*?(\d+).*").sub(r"\1", args[1])
-    elif len(args) == 1 and opts.revision:
-        # accepts -r 3123 http://foo/bar
-        pass
-    elif not opts.list:
-        raise Error, "provide -l or a revision number"
+        # prevent from using the old <name> <rev> syntax
+        try:
+            rev = int(args[1])
+        except ValueError:
+            # ok, it is a package name, let it pass
+            pass
+        else:
+            raise Error, "the format <name> <revision> is deprecated, "\
+                    "use <name>@<revision> instead"
+    opts.urls = [default_parent(nameurl) for nameurl in args]
     return opts
 
 def list_targets(option, opt, val, parser):
@@ -89,9 +92,7 @@ def list_targets(option, opt, val, parser):
     execcmd(command, show=True)
     sys.exit(0)
 
-def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
-    #if not NINZ:
-    #    raise Error, "you must have NINZ installed to use this command"
+def submit(urls, target, define=[], submithost=None):
     if submithost is None:
         submithost = config.get("submit", "host")
         if submithost is None:
@@ -103,13 +104,13 @@ def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
             del type, user, port, path, rest
     # runs a create-srpm in the server through ssh, which will make a
     # copy of the rpm in the export directory
-    if list:
-        raise Error, "unable to list targets from svn+ssh:// URLs"
     createsrpm = get_helper("create-srpm")
-    command = "ssh %s %s '%s' -r %s -t %s" % (
-            submithost, createsrpm, pkgdirurl, revision, target)
-    if define:
-        command += " " + " ".join([ "--define " + x for x in define ])
+    args = ["ssh", submithost, createsrpm, "-t", target]
+    for entry in define:
+        args.append("--define")
+        args.append(entry)
+    args.extend(urls)
+    command = subprocess.list2cmdline(args)
     status, output = execcmd(command)
     if status == 0:
         print "Package submitted!"
@@ -117,7 +118,6 @@ def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
         sys.stderr.write(output)
         sys.exit(status)
 
-
 def main():
     do_command(parse_options, submit)
 
diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py
index c883cd7..b73a9e2 100644
--- a/RepSys/rpmutil.py
+++ b/RepSys/rpmutil.py
@@ -7,6 +7,7 @@ from RepSys.log import specfile_svn2rpm
 from RepSys.util import execcmd
 from RepSys.command import default_parent
 import rpm
+import urlparse
 import tempfile
 import shutil
 import string
@@ -34,6 +35,7 @@ def rpm_macros_defs(macros):
     args = " ".join(defs)
     return args
 
+#FIXME move it to another module
 def rev_touched_url(url, rev):
     svn = SVN()
     info = svn.info2(url)
@@ -50,6 +52,51 @@ def rev_touched_url(url, rev):
             touched = True
     return touched
 
+def svn_url_rev(url, retrieve=True):
+    """Get the revision from a given URL
+
+    If the URL contains an explicit revision number (URL@REV), just use it
+    without even checking if the revision really exists.
+
+    The parameter retrieve defines whether it must ask the SVN server for
+    the revision number or not when it is not found in the URL.
+    """
+    parsed = urlparse.urlparse(url)
+    path = os.path.normpath(parsed[2])
+    dirs = path.rsplit("/", 1)
+    lastname = dirs[-1]
+    index = lastname.rfind("@")
+    rev = None
+    if index != -1:
+        rawrev = lastname[index+1:]
+        if rawrev:
+            try:
+                rev = int(rawrev)
+                if rev < 0:
+                    raise ValueError
+            except ValueError:
+                raise Error, "invalid revision specification on URL: %s" % url
+    if rev is None and retrieve:
+        # if no revspec was found, ask the server
+        svn = SVN()
+        rev = svn.revision(url)
+    return rev
+
+def strip_url_rev(url):
+    """Removes the @REV from a string in the URL@REV scheme"""
+    parsed = list(urlparse.urlparse(url))
+    path = os.path.normpath(parsed[2])
+    dirs = path.rsplit("/", 1)
+    name = lastname = dirs[-1]
+    try:
+        index = lastname.rindex("@")
+    except ValueError:
+        pass
+    else:
+        name = lastname[:index]
+    parsed[2] = os.path.join(dirs[0], name)
+    return urlparse.urlunparse(parsed)
+
 def get_srpm(pkgdirurl,
              mode = "current",
              targetdirs = None,
@@ -81,6 +128,7 @@ def get_srpm(pkgdirurl,
         elif mode == "pristine":
             geturl = os.path.join(pkgdirurl, "pristine")
         elif mode == "current" or mode == "revision":
+            #FIXME we should handle revisions specified using @REV
             geturl = os.path.join(pkgdirurl, "current")
         else:
             raise Error, "unsupported get_srpm mode: %s" % mode
@@ -102,9 +150,6 @@ def get_srpm(pkgdirurl,
             submit = not not revision
             specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
                     template=template, macros=macros, exported=tmpdir)
-        #FIXME revisioreal not needed if revision is None
-        #FIXME use geturl instead of pkgdirurl
-        revisionreal = svn.revision(pkgdirurl)
         for script in scripts:
             #FIXME revision can be "None"
             status, output = execcmd(script, tmpdir, spec, str(revision),
@@ -119,25 +164,30 @@ def get_srpm(pkgdirurl,
             (topdir, builddir, rpmdir, sourcedir, specdir, 
              srcrpmdir, patchdir, packager, spec, defs))
 
-        if revision and revisionreal:
-            #FIXME duplicate glob line
-            srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
-            srpminfo = SRPM(srpm)
-            release = srpminfo.release
-            srpmbase = os.path.basename(srpm)
-            os.rename(srpm, "%s/@%s:%s" % (srpmsdir, revisionreal, srpmbase))
-        srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
+        # copy the generated SRPMs to their target locations
+        targetsrpms = []
+        urlrev = None
+        if revname:
+            urlrev = revision or svn_url_rev(geturl)
         if not targetdirs:
             targetdirs = (".",)
-        targetsrpms = []
-        for targetdir in targetdirs:
-            targetsrpm = os.path.join(os.path.realpath(targetdir), 
-                    os.path.basename(srpm))
-            targetsrpms.append(targetsrpm)
-            if verbose:
-                sys.stderr.write("Wrote: %s\n" %  targetsrpm)
-            execcmd("cp -f", srpm, targetdir)
-        os.unlink(srpm)
+        srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
+        if not srpms:
+            # something fishy happened
+            raise Error, "no SRPMS were found at %s" % srpmsdir
+        for srpm in srpms:
+            name = os.path.basename(srpm)
+            if revname:
+                name = "@%s:%s" % (urlrev, name)
+            for targetdir in targetdirs:
+                newpath = os.path.join(targetdir, name)
+                targetsrpms.append(newpath)
+                if os.path.exists(newpath):
+                    # should we warn?
+                    os.unlink(newpath)
+                shutil.copy(srpm, newpath)
+                if verbose:
+                    sys.stderr.write("Wrote: %s\n" %  newpath)
         return targetsrpms
     finally:
         if os.path.isdir(tmpdir):
diff --git a/create-srpm b/create-srpm
index 0c51e97..868ac27 100755
--- a/create-srpm
+++ b/create-srpm
@@ -1,13 +1,14 @@
 #!/usr/bin/python
 
 from RepSys import Error, config, plugins
-from RepSys.rpmutil import get_srpm
+from RepSys.rpmutil import get_srpm, svn_url_rev, strip_url_rev
 from RepSys.cgiutil import get_targets
 from RepSys.util import mapurl, execcmd, get_helper
 import sys
 import os
 import pwd
 import optparse
+import subprocess
 
 class CmdError(Error): pass
 
@@ -15,15 +16,13 @@ class CmdIface:
     def author_email(self, author):
         return config.get("users", author)
 
-    def submit_package(self, packageurl, packagerev, targetname, 
-            dontmapurl_=0, define=[]):
+    def submit_package(self, urls, revision, targetname, dontmapurl_=0,
+            define=[]):
         pw = pwd.getpwuid(os.getuid())
         username = pw[0]
         packager = config.get("users", username) or pw[4]
         if not packager:
             raise CmdError, "your email was not found"
-        elif not packagerev:
-            raise CmdError, "no revision provided"
         elif not targetname:
             raise CmdError, "no target provided"
         else:
@@ -33,45 +32,48 @@ class CmdIface:
                     break
             else:
                 raise CmdError, "target not found"
-            try:
-                tmp = int(packagerev)
-            except ValueError:
-                raise CmdError, "invalid revision provided"
-            for allowed in target.allowed:
-                if packageurl.startswith(allowed):
-                    break
-            else:
-                raise CmdError, "%s is not allowed for this target" \
-                                % packageurl
-            newurl = packageurl
-            if not dontmapurl_:
-                newurl = mapurl(packageurl)
-            targetsrpms = get_srpm(newurl,
-                     revision=packagerev,
-                     targetdirs=target.target,
-                     packager=packager,
-                     revname=1,
-                     svnlog=1,
-                     scripts=target.scripts, 
-                     macros=target.macros)
-
-            uploadsrpm = get_helper("upload-srpm")
-            if uploadsrpm:
-                upload_command = [ uploadsrpm ]
+            for url in urls:
+                for allowed in target.allowed:
+                    if url.startswith(allowed):
+                        break
+                else:
+                    raise CmdError, "%s is not allowed for this target" \
+                                    % url
+            if not dontmapurl_: #FIXME don't use it!
+                urls = [mapurl(url) for url in urls]
+            uploadsrpms = []
+            for url in urls:
+                revision = revision or svn_url_rev(url)
+                url = strip_url_rev(url)
+                targetsrpms = get_srpm(url,
+                         revision=revision,
+                         targetdirs=target.target,
+                         packager=packager,
+                         svnlog=1,
+                         revname=1,
+                         scripts=target.scripts, 
+                         macros=target.macros)
+                uploadsrpms.extend(targetsrpms)
+            uploadcmd = get_helper("upload-srpm")
+            if uploadcmd:
+                upload_command = [uploadcmd]
                 if define:
                     for x in define:
                         upload_command.append("--define")
                         upload_command.append(x)
                 upload_command.append(targetname)
-                upload_command.append(targetsrpms[0])
-                status, output = execcmd(" ".join(upload_command),
-                        noerror=1)
-                if os.path.isfile(targetsrpms[0]):
-                    os.unlink(targetsrpms[0])
-                else:
-                    sys.stderr.write("warning: upload ok; temp file '%s' removed unexpectedly\n" % (targetsrpms[0]))
+                upload_command.extend(uploadsrpms)
+                command = subprocess.list2cmdline(upload_command)
+                status, output = execcmd(command, noerror=1)
+                for srpm in uploadsrpms:
+                    if os.path.isfile(srpm):
+                        os.unlink(srpm)
+                    else:
+                        sys.stderr.write("warning: temporary file "\
+                                "'%s' removed unexpectedly\n" % srpm)
                 if status != 0:
-                    raise CmdError, "Failed to upload %s:\n%s" % (packageurl, output)
+                    raise CmdError, "Failed to upload "\
+                        "%s:\n%s" % (" ".join(urls), output)
         return 1
 
     def submit_targets(self):
@@ -79,10 +81,8 @@ class CmdIface:
 
 
 def parse_options():
-    usage = "create-srpm <packageurl> -t <target> [-r <revision>]"
+    usage = "create-srpm <packageurl> -t <target>"
     parser = optparse.OptionParser(usage=usage)
-    parser.add_option("-r", "--revision", dest="revision", type="string",
-                      help="the revision number")
     parser.add_option("-t", "--target", type="string", dest="target",
                       help="target name")
     parser.add_option("-M", "--nomapping", action="store_true",
@@ -92,6 +92,8 @@ def parse_options():
     parser.add_option("--list", dest="list_targets", default=False,
                       action="store_true",
                       help="list submit targets available")
+    parser.add_option("-r", help="revision", dest="revision",
+            type="int", default=None)
     opts, args = parser.parse_args()
     if not opts.list_targets and not args:
         parser.error("you must supply a package url")
@@ -107,8 +109,8 @@ def main():
             for target in iface.submit_targets():
                 print target
         else:
-            iface.submit_package(args[0], opts.revision, opts.target,
-                    opts.urlmap, opts.define)
+            iface.submit_package(args, opts.revision, opts.target, opts.urlmap,
+                    opts.define)
     except Error, e:
         sys.stderr.write("error: %s\n" % str(e))
         sys.exit(1)
diff --git a/repsys.8 b/repsys.8
index c7c82bf..54b49cc 100644
--- a/repsys.8
+++ b/repsys.8
@@ -41,7 +41,7 @@ checkout a package
 .IP "\fBci\fP"
 commit changes
 .IP "\fBsubmit\fP"
-submit a package in a given revision for build
+submit a package in a given revision for build and release
 .IP "\fBsync\fP"
 add-remove all file changes from the .spec
 .IP "\fBgetspec\fP"