- don't clone all branches from svn

- make sure to get all revisions when cloning
- further adaptions to make tool more revision control agnostic
- implement support for 'mgarepo up' for git-svn as well

5 files changed, 135 insertions, 49 deletions

diff --git a/MgaRepo/VCS.py b/MgaRepo/VCS.py
index 30d9279..c643cf6 100644
--- a/MgaRepo/VCS.py
+++ b/MgaRepo/VCS.py
@@ -20,39 +20,37 @@ class VCSLogEntry(object):
         return (self.date == other.date)
 
 class VCS(object):
+    vcs_dirname = None
     def __init__(self):
         self.vcs_name = None
         self.vcs_command = None
         self.vcs_wrapper = "mga-ssh"
         self.vcs_supports = {'clone' : False}
+        self.vcs_type = None
         self.env_defaults = None
 
     def _execVcs(self, *args, **kwargs):
         localcmds = ("add", "revert", "cleanup", "mv")
-        kwargs["collecterr"] = False
+        cmd = self.vcs_command + list(args)
+        kwargs["collecterr"] = kwargs.get("collecterr", False)
         if kwargs.get("show"):
             if not kwargs.get("local"):
                kwargs["collecterr"] = True
         else:
-            if args[0] not in localcmds:
-                args = list(args)
-                args.append("--non-interactive")
+            if self.vcs_command is "svn" and args[0] not in localcmds:
+                cmd.append("--non-interactive")
             else:
                 if args[0] == "mv":
                     kwargs["collecterr"] = False
-                else:
-                    kwargs["collecterr"] = True
-        kwargs["cleanerr"] = True
+        kwargs["cleanerr"] = kwargs.get("cleanerr", True)
         if kwargs.get("xml"):
-            args.append("--xml")
-        cmdargs = [self.vcs_command]
-        cmdargs.extend(args)
+            cmd.append("--xml")
         try:
             if args[0] in ('info', 'checkout','log'):
                 kwargs['info'] = True
             else:
                 kwargs['info'] = False
-            return execcmd(cmdargs, **kwargs)
+            return execcmd(*cmd, **kwargs)
         except Error as e:
             msg = None
             if e.args:
diff --git a/MgaRepo/git.py b/MgaRepo/git.py
index c9018e6..a16a1ea 100644
--- a/MgaRepo/git.py
+++ b/MgaRepo/git.py
@@ -14,10 +14,11 @@ class GITLogEntry(VCSLogEntry):
         VCSLogEntry.__init__(self, revision, author, data)
 
 class GIT(VCS):
+    vcs_dirname = ".git"
     def __init__(self):
         VCS.__init__(self)
         self.vcs_name = "git"
-        self.vcs_command = config.get("global", "git-command", "git")
+        self.vcs_command = config.get("global", "git-command", ["git", "svn"])
         self.vcs_supports['clone'] = True
         self.env_defaults = {"GIT_SSH": self.vcs_wrapper}
 
@@ -28,7 +29,8 @@ class GIT(VCS):
             # To speed things up on huge repositories, we'll just grab all the
             # revision numbers for this specific directory and grab these only
             # in stead of having to go through each and every revision...
-            retval, result = execcmd("svn log --stop-on-copy --xml %s" % url)
+            cmd = ["svn", "log", "-g", "--xml", url]
+            retval, result = execcmd(*cmd)
             if retval:
                 return retval
             parser = ElementTree.XMLParser()
@@ -40,16 +42,74 @@ class GIT(VCS):
             topurl = dirname(url)
             trunk = basename(url)
             tags = "releases"
-            execcmd("git svn init %s --trunk=%s --tags=%s %s" % (topurl, trunk, tags, targetpath), show=True)
+            # cloning svn braches as well should rather be optionalif reenabled..
+            #cmd = ["init", topurl, "--trunk="+trunk, "--tags="+tags, targetpath]
+            cmd = ["init", url, targetpath]
+            self._execVcs(*cmd, **kwargs)
             chdir(targetpath)
+            revisions.sort()
             for entry in logentries:
-                revisions.append(entry.attrib["revision"])
+                revisions.append(int(entry.attrib["revision"]))
+            revisions.sort()
             while revisions:
-                execcmd("git svn fetch --log-window-size=1000 -r%d" % int(revisions.pop()), show=True)
-
-            cmd = ["svn", "rebase"]
+                cmd = ["fetch", "--log-window-size=1000", "-r%d" % revisions.pop(0)]
+                self._execVcs(*cmd, **kwargs)
+            cmd = ["rebase", "--log-window-size=1000", "--local", "--fetch-all", "git-svn"]
             return self._execVcs_success(*cmd, **kwargs)
 
+    def update(self, path, **kwargs):
+        cmd = ["log", "--oneline", "--limit=1"]
+        retval, result = self._execVcs(*cmd)
+        if retval:
+            return retval
+
+        revision = result.split()
+
+        if revision[0][0] == 'r':
+            startrev = "-r"+str(int(revision[0][1:])+1)
+        else:
+            startrev = "BASE"
+
+        cmd = ["propget", "svn:entry:committed-rev"]
+        retval, lastrev = self._execVcs(*cmd)
+        if retval:
+            return retval
+
+        cmd = ["git", "config", "--get-regexp", '^svn-remote.svn.(url|fetch)']
+        retval, result = execcmd(*cmd)
+        if retval:
+            return retval
+        result = result.strip().split()
+        url = result[1] + "/" + result[3].split(":")[0]
+
+        # To speed things up on huge repositories, we'll just grab all the
+        # revision numbers for this specific directory and grab these only
+        # in stead of having to go through each and every revision...
+        cmd = ["svn", "log", "-g", "--xml", "%s:%s" % (startrev,lastrev), url]
+        retval, result = execcmd(*cmd)
+        if retval:
+            return retval
+        parser = ElementTree.XMLParser()
+        result = "".join(result.split("\n"))
+        parser.feed(result)
+        log = parser.close()
+        logentries = log.getiterator("logentry")
+        revisions = []
+        chdir(path)
+        for entry in logentries:
+            revisions.append(int(entry.attrib["revision"]))
+        revisions.sort()
+        while revisions:
+            cmd = ["fetch", "--log-window-size=1000", "-r%d" % revisions.pop(0)]
+            self._execVcs(*cmd, **kwargs)
+
+        cmd = ["rebase", "--log-window-size=1000", "--local", "--fetch-all", "git-svn"]
+        status, output = self._execVcs(*cmd, **kwargs)
+        if status == 0:
+            return [x.split() for x in output.split()]
+        return None
+
+
 class SVNLook(VCSLook):
     def __init__(self, repospath, txn=None, rev=None):
         VCSLook.__init__(self, repospath, txn, rev)
diff --git a/MgaRepo/rpmutil.py b/MgaRepo/rpmutil.py
index 850bd4d..207cc3d 100644
--- a/MgaRepo/rpmutil.py
+++ b/MgaRepo/rpmutil.py
@@ -15,8 +15,30 @@ import glob
 import sys
 import os
 
+def detectVCS(url):
+    if ':' in url:
+        protocol,uri = url.split(":")
+        if "svn" in protocol:
+            return SVN()
+        elif "git" in protocol:
+            return GIT()
+        elif "http" in protocol:
+            if uri.endswith(".git"):
+                return GIT()
+            elif "svn" in uri:
+                return SVN()
+            else:
+                raise Error("Unknown protocol %s for %s" % (protocol, url))
+    elif os.path.exists(url) and os.path.isdir(url):
+        if os.path.exists(os.path.join(url,".svn")) and os.path.isdir(os.path.join(url,".svn")):
+            return SVN()
+        if os.path.exists(os.path.join(url,".git")) and os.path.isdir(os.path.join(url,".git")):
+            return GIT()
+    else:
+        raise Error("No supported repository found at path: %s" % url)
+
 def get_spec(pkgdirurl, targetdir=".", submit=False):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     tmpdir = tempfile.mktemp()
     try:
         geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
@@ -36,7 +58,7 @@ def get_spec(pkgdirurl, targetdir=".", submit=False):
 
 #FIXME move it to another module
 def rev_touched_url(url, rev):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     info = svn.info2(url)
     if info is None:
         raise Error("can't fetch svn info about the URL: %s" % url)
@@ -66,7 +88,7 @@ def get_srpm(pkgdirurl,
              macros = [],
              verbose = 0,
              strict = False):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     tmpdir = tempfile.mktemp()
     topdir = "_topdir %s" % tmpdir
     builddir = "_builddir %s/%s" % (tmpdir, "BUILD")
@@ -172,7 +194,7 @@ def get_srpm(pkgdirurl,
 
 def patch_spec(pkgdirurl, patchfile, log=""):
     #FIXME use get_spec
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     tmpdir = tempfile.mktemp()
     try:
         geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
@@ -192,7 +214,7 @@ def patch_spec(pkgdirurl, patchfile, log=""):
 
 def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
         baseurl=None, baseold=None, logmsg=None, rename=True):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     srpm = SRPM(srpmfile)
     tmpdir = tempfile.mktemp()
     if baseurl:
@@ -353,7 +375,7 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
                      (version, srpm.release))
 
 def create_package(pkgdirurl, log="", verbose=0):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     tmpdir = tempfile.mktemp()
     try:
         basename = layout.package_name(pkgdirurl)
@@ -394,7 +416,7 @@ revision: %s
     return log
 
 def mark_release(pkgdirurl, version, release, revision):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     releasesurl = layout.checkout_url(pkgdirurl, releases=True)
     versionurl = "/".join([releasesurl, version])
     releaseurl = "/".join([versionurl, release])
@@ -416,7 +438,7 @@ def mark_release(pkgdirurl, version, release, revision):
              log=markreleaselog)
 
 def check_changed(pkgdirurl, all=0, show=0, verbose=0):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     if all:
         baseurl = pkgdirurl
         packages = []
@@ -487,7 +509,7 @@ def checkout(pkgdirurl, path=None, revision=None, branch=None, distro=None, back
     if path is None:
         path = layout.package_name(pkgdirurl)
     mirror.info(current, write=True)
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     svn.checkout(current, path, rev=revision, show=1)
     if not spec:
         binrepo.download_binaries(path)
@@ -504,6 +526,7 @@ def clone(pkgdirurl, path=None, branch=None,
     git.clone(current, path, show=1)
 
 def getpkgtopdir(basedir=None):
+
     #FIXME this implementation doesn't work well with relative path names,
     # which is something we need in order to have a friendlier output
     if basedir is None:
@@ -520,10 +543,11 @@ def ispkgtopdir(path=None):
     if path is None:
         path = os.getcwd()
     names = os.listdir(path)
-    return (".svn" in names and "SPECS" in names and "SOURCES" in names)
+    vcs = detectVCS(path)
+    return (vcs.vcs_dirname in names and "SPECS" in names and "SOURCES" in names)
 
 def sync(dryrun=False, commit=False, download=False):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     topdir = getpkgtopdir()
     spath = binrepo.sources_path(topdir)
     binrepoentries = binrepo.parse_sources(spath)
@@ -606,7 +630,7 @@ def sync(dryrun=False, commit=False, download=False):
             upload([path], commit=commit)
 
 def commit(target=".", message=None, logfile=None):
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     status = svn.status(target, quiet=True)
     if not status:
         print("nothing to commit")
@@ -641,23 +665,26 @@ def spec_sources(topdir):
     return sources
     
 def update(target=None):
-    svn = SVN()
+    vcs = None
     info = None
-    svn_target = None
+    vcs_target = None
     br_target = None
     if target:
-        svn_target = target
+        vcs_target = target
     else:
         top = getpkgtopdir()
-        svn_target = top
+        vcs_target = top
         br_target = top
-    if svn_target:
-        svn.update(svn_target, show=True)
+    if vcs_target:
+        vcs = detectVCS(vcs_target)
+        vcs.update(vcs_target, show=True)
     if br_target:
-        info = svn.info2(svn_target) 
-        if not br_target and not svn_target:
-            raise Error("target not in SVN nor in binaries "\
-                    "repository: %s" % target)
+        if not vcs:
+            vcs = detectVCS(br_target)
+        info = vcs.info2(vcs_target) 
+        if not br_target and not vcs_target:
+            raise Error("target not in %s nor in binaries "\
+                    "repository: %s" % (type(vcs).__name__,target))
         url = info["URL"]
         binrepo.download_binaries(br_target)
 
@@ -668,12 +695,12 @@ def upload(paths, commit=False):
             binrepo.upload_binary(topdir, os.path.basename(path))
             binrepo.update_sources(topdir, added=[path])
             if commit:
-                svn = SVN()
+                svn = detectVCS(pkgdirurl)
                 silent = config.get("log", "ignore-string", "SILENT")
                 message = "%s: new file %s" % (silent, path)
                 svn.commit(binrepo.sources_path(topdir), log=message)
         else:
-            svn = SVN()
+            svn = detectVCS(pkgdirurl)
             svn.add(path, local=True)
             if commit:
                 silent = config.get("log", "ignore-string", "SILENT")
@@ -688,10 +715,10 @@ def delete(paths, commit=False):
             topdir = getpkgtopdir()
             binrepo.update_sources(topdir, removed=[os.path.basename(path)])
             if commit:
-                svn = SVN()
+                svn = detectVCS(pkgdirurl)
                 svn.commit(binrepo.sources_path(topdir), log=message)
         else:
-            svn = SVN()
+            svn = detectVCS(pkgdirurl)
             svn.remove(path, local=True)
             if commit:
                 svn.commit(path, log=message)
@@ -700,7 +727,7 @@ def obsolete(pkgdirurl, branch=None, distro=None, backports=None, commit=False,
     o_pkgdirurl = pkgdirurl
     pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro, backports=backports)
     pkgdest = layout.package_url(o_pkgdirurl, obsolete=True, backports=backports)
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
     svn.mv(pkgdirurl, pkgdest, message=log)
     if commit:
         svn.commit(path, log=log)
@@ -735,7 +762,7 @@ def get_submit_info(path):
     if not os.path.isdir(os.path.join(path, ".svn")):
         raise Error("subversion directory not found")
     
-    svn = SVN()
+    svn = detectVCS(pkgdirurl)
 
     # Now, extract the package name.
     info = svn.info2(path)
diff --git a/MgaRepo/svn.py b/MgaRepo/svn.py
index 3a74954..120c19c 100644
--- a/MgaRepo/svn.py
+++ b/MgaRepo/svn.py
@@ -12,10 +12,11 @@ class SVNLogEntry(VCSLogEntry):
         VCSLogEntry.__init__(self, revision, author, data)
 
 class SVN(VCS):
+    vcs_dirname = ".svn"
     def __init__(self):
         VCS.__init__(self)
         self.vcs_name = "svn"
-        self.vcs_command = config.get("global", "svn-command", "svn")
+        self.vcs_command = config.get("global", "svn-command", ["svn"])
         self.env_defaults = {"SVN_SSH": self.vcs_wrapper}
 
 class SVNLook(VCSLook):
diff --git a/MgaRepo/util.py b/MgaRepo/util.py
index 057923f..18759aa 100644
--- a/MgaRepo/util.py
+++ b/MgaRepo/util.py
@@ -25,10 +25,10 @@ def execcmd(*cmd, **kwargs):
     assert (kwargs.get("collecterr") and kwargs.get("show")) or not kwargs.get("collecterr"), \
             ("execcmd is implemented to handle collecterr=True only if show=True")
     # split command args
-    if isinstance(cmd[0], str):
+    if isinstance(cmd[0], str) and len(cmd) is 1:
         cmdargs = shlex.split(cmd[0])
     else:
-        cmdargs = cmd[0][:]
+        cmdargs = cmd[:]
 
     stdout = None
     stderr = None