diff options
| -rw-r--r-- | MgaRepo/commands/clone.py | 3 | ||||
| -rw-r--r-- | MgaRepo/git.py | 54 | ||||
| -rw-r--r-- | MgaRepo/log.py | 49 | ||||
| -rw-r--r-- | MgaRepo/rpmutil.py | 4 |
4 files changed, 93 insertions, 17 deletions
diff --git a/MgaRepo/commands/clone.py b/MgaRepo/commands/clone.py index 589ec75..86793c9 100644 --- a/MgaRepo/commands/clone.py +++ b/MgaRepo/commands/clone.py @@ -20,6 +20,7 @@ Options: -b The package branch -M Do not use the mirror (use the main repository) -h Show this message + -F Do not convert svn usernames to full name & email Examples: repsys clone pkgname @@ -33,6 +34,8 @@ def parse_options(): parser = OptionParser(help=HELP) parser.add_option("--distribution", "-d", dest="distro", default=None) parser.add_option("--branch", "-b", dest="branch", default=None) + parser.add_option("-F", dest="fullnames", default=True, + action="store_false") opts, args = parser.parse_args() if len(args) not in (1, 2): raise Error("invalid arguments") diff --git a/MgaRepo/git.py b/MgaRepo/git.py index 45c628d..b91dcb2 100644 --- a/MgaRepo/git.py +++ b/MgaRepo/git.py @@ -2,8 +2,10 @@ from MgaRepo import Error, config from MgaRepo.util import execcmd from MgaRepo.VCS import * from MgaRepo.svn import SVN +from MgaRepo.log import UserTagParser from os.path import basename, dirname, abspath, lexists, join from os import chdir, getcwd +from tempfile import mkstemp import sys import re import time @@ -23,7 +25,23 @@ class GIT(VCS): self.vcs_supports['clone'] = True self.env_defaults = {"GIT_SSH": self.vcs_wrapper} - def clone(self, url, targetpath, **kwargs): + def configget(self, key="", location="--local"): + cmd = ["config", location, "--get-regexp", key] + config = None + status, output = self._execVcs(*cmd, noerror=True) + if not status and output: + config = eval("{'" + output.replace("\n", "',\n'").replace(" ", "' : '") + "'}") + return config + + def configset(self, config, location="--local"): + cmd = ("config", location) + for pair in config.items(): + status, output = self._execVcs(*cmd + pair) + if status: + return False + return True + + def clone(self, url, targetpath, fullnames=True, **kwargs): if lexists(join(targetpath, SVN.vcs_dirname)): raise Error("Target path %s already contains svn checkout, aborting..." % targetpath) if url.split(':')[0].find("svn") < 0: @@ -44,15 +62,31 @@ class GIT(VCS): tags = "releases" # cloning svn braches as well should rather be optionalif reenabled.. #cmd = ["svn", "init", topurl, "--trunk="+trunk, "--tags="+tags", targetpath] + + cmd = ["svn", "init", url, abspath(targetpath)] self._execVcs(*cmd, **kwargs) os.environ.update({"GIT_WORK_TREE" : abspath(targetpath), "GIT_DIR" : join(abspath(targetpath),".git")}) + for entry in logentries: revisions.append(int(entry.attrib["revision"])) revisions.sort() + + fetchcmd = ["svn", "fetch", "--log-window-size=1000"] + if fullnames: + usermap = UserTagParser() + # store configuration in local git config so that'll be reused later when ie. updating + gitconfig = {"svn-remote.authorlog.url" : usermap.url, + "svn-remote.authorlog.defaultmail": usermap.defaultmail} + self.configset(gitconfig) + usermapfile = usermap.get_user_map_file() + fetchcmd.extend(("--authors-file", usermapfile)) + while revisions: - cmd = ["svn", "fetch", "--log-window-size=1000", "-r%d" % revisions.pop(0)] - self._execVcs(*cmd, **kwargs) + self._execVcs(*fetchcmd + ["-r%d"%revisions.pop(0)], **kwargs) + if fullnames: + usermap.cleanup() + cmd = ["svn", "rebase", "--log-window-size=1000", "--local", "--fetch-all", "git-svn"] return self._execVcs_success(*cmd, **kwargs) @@ -114,15 +148,25 @@ class GIT(VCS): retval, result = execcmd(*cmd) if retval: return retval + xmllog = ElementTree.fromstring(result) logentries = xmllog.getiterator("logentry") revisions = [] for entry in logentries: revisions.append(int(entry.attrib["revision"])) revisions.sort() + + fetchcmd = ["svn", "fetch", "--log-window-size=1000"] + gitconfig = self.configget("svn-remote.authorlog") + if gitconfig: + usermap = UserTagParser(url=gitconfig.get("svn-remote.authorlog.url"),defaultmail=gitconfig.get("svn-remote.authorlog.defaultmail")) + usermapfile = usermap.get_user_map_file() + fetchcmd.extend(("--authors-file", usermapfile)) + while revisions: - cmd = ["svn", "fetch", "--log-window-size=1000", "-r%d" % revisions.pop(0)] - self._execVcs(*cmd, **kwargs) + self._execVcs(*fetchcmd + ["-r%d"%revisions.pop(0)], **kwargs) + if gitconfig: + usermap.cleanup() cmd = ["svn", "rebase", "--log-window-size=1000", "--local", "--fetch-all", "git-svn"] status, output = self._execVcs(*cmd, **kwargs) diff --git a/MgaRepo/log.py b/MgaRepo/log.py index 5c2a56d..3fb3cc5 100644 --- a/MgaRepo/log.py +++ b/MgaRepo/log.py @@ -8,6 +8,7 @@ from io import StringIO import sys import os +import os.path import re import time import locale @@ -562,11 +563,19 @@ def get_old_log(pkgdirurl): return chlog from html.parser import HTMLParser -class TagParser(HTMLParser): +from urllib.request import urlopen +class UserTagParser(HTMLParser): li = False ahref = False userpage = None namepat = re.compile("(?P<name>.*?)\s*\((?P<user>.*?)\)") + usermap = {} + usermapfile = None + + def __init__(self, url=None, defaultmail=None, *cmd, **kwargs): + HTMLParser.__init__(self, *cmd, **kwargs) + self.url = url or "http://people.mageia.org/u/" + self.defaultmail = defaultmail or "mageia.org" def handle_starttag(self, tag, attrs): if tag == "li": @@ -587,17 +596,36 @@ class TagParser(HTMLParser): def handle_data(self, data): if self.li and self.ahref: found = self.namepat.match(data) - if found and found.group("user") and found.group("name") and found.group("user")+".html" == self.userpage: - usermap[found.group("user")] = "%s <%s@mageia.org>" % (found.group("name"), found.group("user")) + if found: + user = found.group("user") + name = found.group("name") + if user and name and user+".html" == self.userpage: + self.usermap[user] = "%s <%s@%s>" % (name, user, self.defaultmail) + + def get_user_map(self): + f = urlopen(self.url) + userhtml = f.read().decode("UTF-8") + f.close() + self.feed(userhtml) + return self.usermap + + def get_user_map_file(self): + if not self.usermap: + self.get_user_map() + self.usermapfile = tempfile.mkstemp(suffix=".txt", prefix="usermap") + f = open(self.usermapfile[0], "w", encoding="UTF-8") + f.writelines("%s = %s\n" % user for user in sorted(self.usermap.items())) + f.close() + return self.usermapfile[1] + + def cleanup(self): + if os.path.exists(self.usermapfile[1]): + os.unlink(self.usermapfile[1]) def _map_user_names(): - import urllib.request if not usermap: - f = urllib.request.urlopen("https://people.mageia.org/u/") - users = f.read().decode("UTF-8") - f.close() - parser = TagParser() - parser.feed(users) + parser = UserTagParser() + usermap.update(parser.get_user_map()) def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, submit=False, sort=False, template=None, macros=[], exported=None, @@ -627,7 +655,8 @@ def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, newlog = StringIO() if svn: if fullnames: - _map_user_names() + if not usermap: + _map_user_names() rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, template=template, macros=macros, exported=exported) newlog.write(rawsvnlog) diff --git a/MgaRepo/rpmutil.py b/MgaRepo/rpmutil.py index bb9964a..3a10380 100644 --- a/MgaRepo/rpmutil.py +++ b/MgaRepo/rpmutil.py @@ -591,7 +591,7 @@ def checkout(pkgdirurl, path=None, revision=None, branch=None, distro=None, back binrepo.download_binaries(path) def clone(pkgdirurl, path=None, revision=None, branch=None, distro=None, backports=None, - spec=False): + spec=False, fullnames = True): o_pkgdirurl = pkgdirurl pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro, backports=backports) append = None @@ -603,7 +603,7 @@ def clone(pkgdirurl, path=None, revision=None, branch=None, distro=None, backpor path = layout.package_name(pkgdirurl) mirror.info(current, write=True) git = GIT() - git.clone(current, path, show=1) + git.clone(current, path, fullnames=fullnames, show=1) if not spec: binrepo.download_binaries(path) |