make the V1_6_X branch the trunk

1 files changed, 216 insertions, 115 deletions

diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py
index 4cbba62..0ca6bdb 100644
--- a/RepSys/rpmutil.py
+++ b/RepSys/rpmutil.py
@@ -1,15 +1,15 @@
 #!/usr/bin/python
-from RepSys import Error, config, RepSysTree
-from RepSys import mirror
+from RepSys import Error, config
+from RepSys import mirror, layout, log
 from RepSys.svn import SVN
 from RepSys.simplerpm import SRPM
-from RepSys.log import specfile_svn2rpm
 from RepSys.util import execcmd
-import pysvn
 from RepSys.command import default_parent
 import rpm
+import urlparse
 import tempfile
 import shutil
+import string
 import glob
 import sys
 import os
@@ -18,8 +18,8 @@ def get_spec(pkgdirurl, targetdir=".", submit=False):
     svn = SVN()
     tmpdir = tempfile.mktemp()
     try:
-        geturl = "/".join([pkgdirurl, "current", "SPECS"])
-        svn.export("%s" % geturl, tmpdir)
+        geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
+        svn.export("'%s'" % geturl, tmpdir)
         speclist = glob.glob(os.path.join(tmpdir, "*.spec"))
         if not speclist:
             raise Error, "no spec files found"
@@ -34,6 +34,23 @@ def rpm_macros_defs(macros):
     args = " ".join(defs)
     return args
 
+#FIXME move it to another module
+def rev_touched_url(url, rev):
+    svn = SVN()
+    info = svn.info2(url)
+    if info is None:
+        raise Error, "can't fetch svn info about the URL: %s" % url
+    root = info["Repository Root"]
+    urlpath = url[len(root):]
+    touched = False
+    entries = svn.log(root, start=rev, limit=1)
+    entry = entries[0]
+    for change in entry.changed:
+        path = change.get("path")
+        if path and path.startswith(urlpath):
+            touched = True
+    return touched
+
 def get_srpm(pkgdirurl,
              mode = "current",
              targetdirs = None,
@@ -47,7 +64,8 @@ def get_srpm(pkgdirurl,
              submit = False,
              template = None,
              macros = [],
-             verbose = 0):
+             verbose = 0,
+             strict = False):
     svn = SVN()
     tmpdir = tempfile.mktemp()
     topdir = "--define '_topdir %s'" % tmpdir
@@ -57,17 +75,26 @@ def get_srpm(pkgdirurl,
     specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS")
     srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS")
     patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES")
+
     try:
         if mode == "version":
-            geturl = os.path.join(pkgdirurl, "releases",
-                                  version, release)
+            geturl = layout.checkout_url(pkgdirurl, version=version,
+                    release=release)
         elif mode == "pristine":
-            geturl = os.path.join(pkgdirurl, "pristine")
+            geturl = layout.checkout_url(pkgdirurl, pristine=True)
         elif mode == "current" or mode == "revision":
-            geturl = os.path.join(pkgdirurl, "current")
+            #FIXME we should handle revisions specified using @REV
+            geturl = layout.checkout_url(pkgdirurl)
         else:
             raise Error, "unsupported get_srpm mode: %s" % mode
-        exportedrev = svn.export(geturl, tmpdir, revision=SVN.makerev(revision))
+        strict = strict or config.getbool("submit", "strict-revision", False)
+        if strict and not rev_touched_url(geturl, revision):
+            #FIXME would be nice to have the revision number even when
+            # revision is None
+            raise Error, "the revision %s does not change anything "\
+                    "inside %s" % (revision or "HEAD", geturl)
+        mirror.info(geturl)
+        svn.export(geturl, tmpdir, rev=revision)
         srpmsdir = os.path.join(tmpdir, "SRPMS")
         os.mkdir(srpmsdir)
         specsdir = os.path.join(tmpdir, "SPECS")
@@ -77,7 +104,7 @@ def get_srpm(pkgdirurl,
         spec = speclist[0]
         if svnlog:
             submit = not not revision
-            specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
+            log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
                     template=template, macros=macros, exported=tmpdir)
         for script in scripts:
             #FIXME revision can be "None"
@@ -89,39 +116,46 @@ def get_srpm(pkgdirurl,
             packager = " --define 'packager %s'" % packager
 
         defs = rpm_macros_defs(macros)
-        execcmd("rpm -bs --nodeps %s %s %s %s %s %s %s %s %s %s" %
-            (topdir, builddir, rpmdir, sourcedir, specdir, 
-             srcrpmdir, patchdir, packager, spec, defs))
-
-        if revision:
-            #FIXME duplicate glob line
-            srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
-            srpminfo = SRPM(srpm)
-            release = srpminfo.release
-            srpmbase = os.path.basename(srpm)
-            os.rename(srpm, "%s/@%s:%s" % (srpmsdir, exportedrev.number, srpmbase))
-        srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
+        sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
+        execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" %
+            (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir,
+                srcrpmdir, patchdir, packager, spec, defs))
+
+        # copy the generated SRPMs to their target locations
+        targetsrpms = []
+        urlrev = None
+        if revname:
+            urlrev = revision or layout.get_url_revision(geturl)
         if not targetdirs:
             targetdirs = (".",)
-        targetsrpms = []
-        for targetdir in targetdirs:
-            targetsrpm = os.path.join(os.path.realpath(targetdir), 
-                    os.path.basename(srpm))
-            targetsrpms.append(targetsrpm)
-            if verbose:
-                sys.stderr.write("Wrote: %s\n" %  targetsrpm)
-            execcmd("cp -f", srpm, targetdir)
-        os.unlink(srpm)
+        srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
+        if not srpms:
+            # something fishy happened
+            raise Error, "no SRPMS were found at %s" % srpmsdir
+        for srpm in srpms:
+            name = os.path.basename(srpm)
+            if revname:
+                name = "@%s:%s" % (urlrev, name)
+            for targetdir in targetdirs:
+                newpath = os.path.join(targetdir, name)
+                targetsrpms.append(newpath)
+                if os.path.exists(newpath):
+                    # should we warn?
+                    os.unlink(newpath)
+                shutil.copy(srpm, newpath)
+                if verbose:
+                    sys.stderr.write("Wrote: %s\n" %  newpath)
         return targetsrpms
     finally:
         if os.path.isdir(tmpdir):
             shutil.rmtree(tmpdir)
 
 def patch_spec(pkgdirurl, patchfile, log=""):
+    #FIXME use get_spec
     svn = SVN()
     tmpdir = tempfile.mktemp()
     try:
-        geturl = "/".join([pkgdirurl, "current", "SPECS"])
+        geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
         svn.checkout(geturl, tmpdir)
         speclist = glob.glob(os.path.join(tmpdir, "*.spec"))
         if not speclist:
@@ -131,39 +165,46 @@ def patch_spec(pkgdirurl, patchfile, log=""):
         if status != 0:
             raise Error, "can't apply patch:\n%s\n" % output
         else:
-            svn.checkin(tmpdir, log=log)
+            svn.commit(tmpdir, log="")
     finally:
         if os.path.isdir(tmpdir):
             shutil.rmtree(tmpdir)
 
-def put_srpm(pkgdirurl, srpmfile, appendname=0, log=""):
-    srpm = SRPM(srpmfile)
-    if appendname:
-        pkgdirurl = "/".join([pkgdirurl, srpm.name])
+def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
+        baseurl=None, baseold=None, logmsg=None):
     svn = SVN()
+    srpm = SRPM(srpmfile)
     tmpdir = tempfile.mktemp()
+    if baseurl:
+        pkgurl = mirror._joinurl(baseurl, srpm.name)
+    else:
+        pkgurl = layout.package_url(srpm.name, distro=branch,
+                mirrored=False)
+    print "Importing package to %s" % pkgurl
     try:
         if srpm.epoch:
             version = "%s:%s" % (srpm.epoch, srpm.version)
         else:
             version = srpm.version
-        versionurl = "/".join([pkgdirurl, "releases", version])
+        versionurl = "/".join([pkgurl, "releases", version])
         releaseurl = "/".join([versionurl, srpm.release])
-        ret = svn.mkdir(pkgdirurl, "Created package directory", noerror=1)
-        if ret:
-            svn.checkout(pkgdirurl, tmpdir)
+        currenturl = os.path.join(tmpdir, "current")
+        #FIXME when pre-commit hook fails, there's no clear way to know
+        # what happened
+        ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory")
+        if ret or not svn.ls(currenturl, noerror=1):
+            svn.checkout(pkgurl, tmpdir)
             svn.mkdir(os.path.join(tmpdir, "releases"))
-            svn.mkdir(os.path.join(tmpdir, "releases", version))
-            svn.mkdir(os.path.join(tmpdir, "current"))
-            svn.mkdir(os.path.join(tmpdir, "current", "SPECS"))
-            svn.mkdir(os.path.join(tmpdir, "current", "SOURCES"))
+            svn.mkdir(currenturl)
+            svn.mkdir(os.path.join(currenturl, "SPECS"))
+            svn.mkdir(os.path.join(currenturl, "SOURCES"))
             #svn.commit(tmpdir,log="Created package structure.")
             version_exists = 1
             currentdir = os.path.join(tmpdir, "current")
         else:
-            if svn.exists(releaseurl):
+            if svn.ls(releaseurl, noerror=1):
                 raise Error, "release already exists"
-            svn.checkout("/".join([pkgdirurl, "current"]), tmpdir)
+            svn.checkout("/".join([pkgurl, "current"]), tmpdir)
             svn.mkdir(versionurl, noerror=1,
                       log="Created directory for version %s." % version)
             currentdir = tmpdir
@@ -217,28 +258,56 @@ def put_srpm(pkgdirurl, srpmfile, appendname=0, log=""):
             if os.path.isdir(unpackdir):
                 shutil.rmtree(unpackdir)
 
-        svn.checkin(tmpdir, log=log)
+        if striplog:
+            specs = glob.glob(os.path.join(specsdir, "*.spec"))
+            if not specs:
+                raise Error, "no spec file fount on %s" % specsdir
+            specpath = specs[0]
+            fspec = open(specpath)
+            spec, chlog = log.split_spec_changelog(fspec)
+            chlog.seek(0)
+            fspec.close()
+            oldurl = baseold or config.get("log", "oldurl")
+            pkgoldurl = mirror._joinurl(oldurl, srpm.name)
+            svn.mkdir(pkgoldurl, noerror=1,
+                    log="created old log directory for %s" % srpm.name)
+            logtmp = tempfile.mktemp()
+            try:
+                svn.checkout(pkgoldurl, logtmp)
+                miscpath = os.path.join(logtmp, "log")
+                fmisc = open(miscpath, "w+")
+                fmisc.writelines(chlog)
+                fmisc.close()
+                svn.add(miscpath)
+                svn.commit(logtmp,
+                        log="imported old log for %s" % srpm.name)
+            finally:
+                if os.path.isdir(logtmp):
+                    shutil.rmtree(logtmp)
+        svn.commit(tmpdir,
+                log=logmsg or ("imported package %s" % srpm.name))
     finally:
         if os.path.isdir(tmpdir):
             shutil.rmtree(tmpdir)
 
     # Do revision and pristine tag copies
-    pristineurl = os.path.join(pkgdirurl, "pristine")
+    pristineurl = layout.checkout_url(pkgurl, pristine=True)
     svn.remove(pristineurl, noerror=1,
                log="Removing previous pristine/ directory.")
-    currenturl = os.path.join(pkgdirurl, "current")
+    currenturl = layout.checkout_url(pkgurl)
     svn.copy(currenturl, pristineurl,
              log="Copying release %s-%s to pristine/ directory." %
                  (version, srpm.release))
-    svn.copy(currenturl, releaseurl,
-             log="Copying release %s-%s to releases/ directory." %
-                 (version, srpm.release))
+    if markrelease:
+        svn.copy(currenturl, releaseurl,
+                 log="Copying release %s-%s to releases/ directory." %
+                     (version, srpm.release))
 
 def create_package(pkgdirurl, log="", verbose=0):
     svn = SVN()
     tmpdir = tempfile.mktemp()
     try:
-        basename = RepSysTree.pkgname(pkgdirurl)
+        basename = layout.package_name(pkgdirurl)
         if verbose:
             print "Creating package directory...",
         sys.stdout.flush()
@@ -257,7 +326,7 @@ def create_package(pkgdirurl, log="", verbose=0):
         if verbose:
             print "done"
             print "Committing...",
-        svn.checkin(tmpdir,
+        svn.commit(tmpdir,
                    log="Created package structure for '%s'." % basename)
         print "done"
     finally:
@@ -280,21 +349,21 @@ def mark_release(pkgdirurl, version, release, revision):
     releasesurl = "/".join([pkgdirurl, "releases"])
     versionurl = "/".join([releasesurl, version])
     releaseurl = "/".join([versionurl, release])
-    if svn.exists(releaseurl, noerror=1):
+    if svn.ls(releaseurl, noerror=1):
         raise Error, "release already exists"
     svn.mkdir(releasesurl, noerror=1,
               log="Created releases directory.")
     svn.mkdir(versionurl, noerror=1,
               log="Created directory for version %s." % version)
-    pristineurl = os.path.join(pkgdirurl, "pristine")
+    pristineurl = layout.checkout_url(pkgdirurl, pristine=True)
     svn.remove(pristineurl, noerror=1,
                log="Removing previous pristine/ directory.")
-    currenturl = os.path.join(pkgdirurl, "current")
+    currenturl = layout.checkout_url(pkgdirurl)
     svn.copy(currenturl, pristineurl,
              log="Copying release %s-%s to pristine/ directory." %
                  (version, release))
     markreleaselog = create_markrelease_log(version, release, revision)
-    svn.copy(currenturl, releaseurl, revision=revision,
+    svn.copy(currenturl, releaseurl, rev=revision,
              log=markreleaselog)
 
 def check_changed(pkgdirurl, all=0, show=0, verbose=0):
@@ -305,29 +374,30 @@ def check_changed(pkgdirurl, all=0, show=0, verbose=0):
         if verbose:
             print "Getting list of packages...",
             sys.stdout.flush()
-        packages = [x['name'] for x in svn.ls(baseurl)]
+        packages = [x[:-1] for x in svn.ls(baseurl)]
         if verbose:
             print "done"
         if not packages:
             raise Error, "couldn't get list of packages"
     else:
-        packages = [pkgdirurl]
+        baseurl, basename = os.path.split(pkgdirurl)
+        packages = [basename]
     clean = []
     changed = []
     nopristine = []
     nocurrent = []
-    for pkgdirurl in packages:
-        package = os.path.basename(pkgdirurl)
-        current = os.path.join(pkgdirurl, "current")
-        pristine = os.path.join(pkgdirurl, "pristine")
+    for package in packages:
+        pkgdirurl = os.path.join(baseurl, package)
+        current = layout.checkout_url(pkgdirurl)
+        pristine = layout.checkout_url(pkgdirurl, pristine=True)
         if verbose:
             print "Checking package %s..." % package,
             sys.stdout.flush()
-        if not svn.exists(current):
+        if not svn.ls(current, noerror=1):
             if verbose:
                 print "NO CURRENT"
             nocurrent.append(package)
-        elif not svn.exists(pristine):
+        elif not svn.ls(pristine, noerror=1):
             if verbose:
                 print "NO PRISTINE"
             nopristine.append(package)
@@ -356,19 +426,16 @@ def check_changed(pkgdirurl, all=0, show=0, verbose=0):
             "nocurrent": nocurrent,
             "nopristine": nopristine}
 
-def checkout(pkgdirurl, path=None, revision=None, use_mirror=True):
+def checkout(pkgdirurl, path=None, revision=None, branch=None,
+        distro=None):
     o_pkgdirurl = pkgdirurl
-    pkgdirurl = default_parent(o_pkgdirurl)
-    current = os.path.join(pkgdirurl, "current")
+    pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro)
+    current = layout.checkout_url(pkgdirurl, branch=branch)
     if path is None:
-        _, path = os.path.split(pkgdirurl)
-    # if default_parent changed the URL, we can use mirrors because the
-    # user did not provided complete package URL
-    if (o_pkgdirurl != pkgdirurl) and use_mirror and mirror.enabled():
-        current = mirror.checkout_url(current)
-        print "checking out from mirror", current
+        path = layout.package_name(pkgdirurl)
+    mirror.info(current)
     svn = SVN()
-    svn.checkout(current, path, revision=SVN.makerev(revision), show=1)
+    svn.checkout(current, path, rev=revision, show=1)
 
 def _getpkgtopdir(basedir=None):
     if basedir is None:
@@ -378,15 +445,15 @@ def _getpkgtopdir(basedir=None):
     if dirname == "SPECS" or dirname == "SOURCES":
         topdir = os.pardir
     else:
-        topdir = "."
+        topdir = ""
     return topdir
 
-def sync(dryrun=False):
+def sync(dryrun=False, download=False):
     svn = SVN()
     topdir = _getpkgtopdir()
     # run svn info because svn st does not complain when topdir is not an
     # working copy
-    svn.info(topdir)
+    svn.info(topdir or ".")
     specsdir = os.path.join(topdir, "SPECS/")
     sourcesdir = os.path.join(topdir, "SOURCES/")
     for path in (specsdir, sourcesdir):
@@ -401,18 +468,34 @@ def sync(dryrun=False):
         spec = rpm.TransactionSet().parseSpec(specpath)
     except rpm.error, e:
         raise Error, "could not load spec file: %s" % e
-    sources = [os.path.basename(name)
-            for name, no, flags in spec.sources()]
-    sourcesst = dict((os.path.basename(st.path), st)
-            for st in svn.status(sourcesdir, get_all=False, ignore=True))
+    sources = dict((os.path.basename(name), name)
+            for name, no, flags in spec.sources())
+    sourcesst = dict((os.path.basename(path), (path, st))
+            for st, path in svn.status(sourcesdir, noignore=True))
     toadd = []
-    for source in sources:
+    for source, url in sources.iteritems():
         sourcepath = os.path.join(sourcesdir, source)
-        if sourcesst.get(source):
+        pst = sourcesst.get(source)
+        if pst:
+            if os.path.isfile(sourcepath):
+                toadd.append(sourcepath)
+            else:
+                sys.stderr.write("warning: %s not found, skipping\n" % sourcepath)
+        elif download and not os.path.isfile(sourcepath):
+            print "%s not found, downloading from %s" % (sourcepath, url)
+            fmt = config.get("global", "download-command",
+                    "wget -c -O '$dest' $url")
+            context = {"dest": sourcepath, "url": url}
+            try:
+                cmd = string.Template(fmt).substitute(context)
+            except KeyError, e:
+                raise Error, "invalid variable %r in download-command "\
+                        "configuration option" % e
+            execcmd(cmd, show=True)
             if os.path.isfile(sourcepath):
                 toadd.append(sourcepath)
             else:
-                sys.stderr.write("warning: %s not found\n" % sourcepath)
+                raise Error, "file not found: %s" % sourcepath
     # rm entries not found in sources and still in svn
     found = os.listdir(sourcesdir)
     toremove = []
@@ -426,31 +509,35 @@ def sync(dryrun=False):
     for path in toremove:
         print "D\t%s" % path
         if not dryrun:
-            svn.remove(path)
+            svn.remove(path, local=True)
     for path in toadd:
         print "A\t%s" % path
         if not dryrun:
-            svn.add(path)
+            svn.add(path, local=True)
 
-def commit(target=".", message=None):
+def commit(target=".", message=None, logfile=None):
     svn = SVN()
-    status = svn.status(target, silent=True)
+    status = svn.status(target, quiet=True)
     if not status:
         print "nothing to commit"
         return
-    info = svn.info(target)
-    url = info.url
+    info = svn.info2(target)
+    url = info.get("URL")
     if url is None:
         raise Error, "working copy URL not provided by svn info"
-    mirrored = mirror.enabled(url)
+    mirrored = mirror.using_on(url)
     if mirrored:
         newurl = mirror.switchto_parent(svn, url, target)
         print "relocated to", newurl
-    # we can't use the svn object here because pexpect hides VISUAL
-    mopt = ""
+    # we can't use the svn object here because svn --non-interactive option
+    # hides VISUAL
+    opts = []
     if message is not None:
-        mopt = "-m \"%s\"" % message
-    os.system("svn ci %s %s" % (mopt, target))
+        opts.append("-m \"%s\"" % message)
+    if logfile is not None:
+        opts.append("-F \"%s\"" % logfile)
+    mopts = " ".join(opts)
+    os.system("svn ci %s %s" % (mopts, target))
     if mirrored:
         print "use \"repsys switch\" in order to switch back to mirror "\
                 "later"
@@ -458,8 +545,10 @@ def commit(target=".", message=None):
 def switch(mirrorurl=None):
     svn  = SVN()
     topdir = _getpkgtopdir()
-    info = svn.info(topdir)
-    wcurl = info.url
+    info = svn.info2(topdir)
+    wcurl = info.get("URL")
+    if wcurl is None:
+        raise Error, "working copy URL not provided by svn info"
     newurl = mirror.autoswitch(svn, topdir, wcurl, mirrorurl)
     print "switched to", newurl
 
@@ -483,31 +572,43 @@ def get_submit_info(path):
     if not os.path.isdir(os.path.join(path, ".svn")):
         raise Error, "subversion directory not found"
     
-    # Now, extract the package name.
     svn = SVN()
-    info = svn.info(path)
 
-    url = info.url
+    # Now, extract the package name.
+    info = svn.info2(path)
+    url = info.get("URL")
+    if url is None:
+        raise Error, "missing URL from svn info %s" % path
     toks = url.split("/")
     if len(toks) < 2 or toks[-1] != "current":
-        raise Error, "invalid package URL %s, needs current/" % url
+        raise Error, "unexpected URL received from 'svn info'"
     name = toks[-2]
+    url = "/".join(toks[:-1])
 
     # Finally, guess revision.
     max = -1
     files = []
     files.extend(glob.glob("%s/*" % specsdir))
     files.extend(glob.glob("%s/*" % sourcesdir))
-    for file in files: 
-        info = svn.info(file)
-        if info is None: # not in working copy
+    for file in files:
+        try:
+            info = svn.info2(file)
+        except Error:
+            # possibly not tracked
             continue
-        rev = info.commit_revision.number
-        if rev > max:
-            max = rev
+        if info is None:
+            continue
+        rawrev = info.get("Last Changed Rev")
+        if rawrev:
+            rev = int(rawrev)
+            if rev > max:
+                max = rev
     if max == -1:
-        raise Error, "unable to find the latest revision"
+        raise Error, "revision tag not found in 'svn info' output"
+
+    if mirror.using_on(url):
+        url = mirror.switchto_parent_url(url)
     
-    return name, max
+    return name, url, max
 
 # vim:et:ts=4:sw=4