aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--MgaRepo/GitHub.py89
-rw-r--r--MgaRepo/VCS.py470
-rw-r--r--MgaRepo/binrepo.py4
-rw-r--r--MgaRepo/commands/buildrpm.py45
-rw-r--r--MgaRepo/commands/clone.py54
-rw-r--r--MgaRepo/commands/getsrpm.py3
-rw-r--r--MgaRepo/commands/github.py57
-rw-r--r--MgaRepo/commands/log.py12
-rw-r--r--MgaRepo/commands/maintdb.py3
-rw-r--r--MgaRepo/commands/rpmlog.py10
-rw-r--r--MgaRepo/commands/submit.py10
-rw-r--r--MgaRepo/git.py192
-rw-r--r--MgaRepo/log.py129
-rw-r--r--MgaRepo/rpmutil.py314
-rw-r--r--MgaRepo/simplerpm.py12
-rw-r--r--MgaRepo/svn.py452
-rw-r--r--MgaRepo/util.py139
-rwxr-xr-xcreate-srpm4
-rwxr-xr-xmgarepo1
-rw-r--r--mgarepo.conf4
-rwxr-xr-xsetup.py5
21 files changed, 1382 insertions, 627 deletions
diff --git a/MgaRepo/GitHub.py b/MgaRepo/GitHub.py
new file mode 100644
index 0000000..71f53d6
--- /dev/null
+++ b/MgaRepo/GitHub.py
@@ -0,0 +1,89 @@
+from MgaRepo import Error, config
+from MgaRepo.rpmutil import detectVCS, get_pkg_tag, clone
+from MgaRepo.util import execcmd
+from MgaRepo import layout
+from MgaRepo.git import GIT
+from MgaRepo.svn import SVN
+from rpm import RPMTAG_SUMMARY, RPMTAG_URL
+import github
+import os
+
+class GitHub(object):
+ def __init__(self, username = config.get("github", "login"), password = config.get("github", "password")):
+ self._github = github.Github(login_or_token=username, password=password)
+ self._organization = self._github.get_organization(config.get("github", "organization", "mdkcauldron"))
+ self._repos = self._organization.get_repos()
+
+ def repository_exists(self, name):
+ for repo in self._repos:
+ if repo.name == name:
+ return repo
+ return None
+
+ def create_repository(self, pkgname, **kwargs):
+ repository = self._organization.create_repo(pkgname, **kwargs)
+ return repository
+
+ def delete_repository(self, pkgname, **kwargs):
+ repository = self.repository_exists(pkgname)
+ if repository:
+ print("deleting repository %s" % repository.full_name)
+ repository.delete()
+ return True
+ raise Error("repository %s doesn't exist!" % (self._organization.login+"/"+pkgname))
+
+ def import_package(self, target):
+ if not os.path.exists(target):
+ target = layout.checkout_url(layout.package_url(target))
+ vcs = detectVCS(target)
+ top_dir = vcs.get_topdir()
+ pkgname = layout.package_name(layout.remove_current(vcs.url))
+
+ repository = self.repository_exists(pkgname)
+ if not repository or not repository.get_stats_commit_activity():
+ if not repository:
+ if os.path.exists(vcs.path):
+ summary = get_pkg_tag(RPMTAG_SUMMARY, path=top_dir)
+ url = get_pkg_tag(RPMTAG_URL, path=top_dir)
+ repository = self.create_repository(pkgname, description=summary, homepage=url)
+ print("GitHub repository created at " + repository.html_url)
+ else:
+ print("Empty GitHub repository already created at %s, using" % repository.html_url)
+
+ if isinstance(vcs, GIT):
+ status, output = vcs.remote("add", repository.full_name, repository.ssh_url, noerror=True)
+ if status:
+ if status == 128 and ("fatal: remote %s already exists." % repository.full_name) \
+ in output:
+ pass
+ else:
+ raise Error(output)
+
+ status, output = vcs.push("--mirror", repository.full_name, show=True)
+ if status == 0:
+ print("Success!")
+ return True
+ elif isinstance(vcs, SVN):
+ clone(vcs.url, bindownload=False)
+ return self.import_package(pkgname)
+
+ else:
+ raise Error("GitHub repository already exists at " + repository.html_url)
+ raise Error("GitHub import failed...")
+
+ def clone_repository(self, pkgname, target=None):
+ if not target:
+ target = pkgname
+ repository = self.repository_exists(pkgname)
+ if repository:
+ svnurl = layout.checkout_url(layout.package_url(pkgname))
+ if repository.permissions:
+ giturl = repository.ssh_url
+ else:
+ giturl = repository.git_url
+ execcmd(("git", "clone", "--mirror", giturl, os.path.join(target, ".git")), show=True)
+ git_svn = GIT(path=target, url=svnurl)
+ git_svn.init(svnurl, pkgname, branch="master", fullnames=True)
+
+ return True
+ raise Error("Repository %s doesn't exist!" % (self._organization.login+"/"+pkgname))
diff --git a/MgaRepo/VCS.py b/MgaRepo/VCS.py
new file mode 100644
index 0000000..ee419f4
--- /dev/null
+++ b/MgaRepo/VCS.py
@@ -0,0 +1,470 @@
+from MgaRepo import Error, SilentError, config
+from MgaRepo.util import execcmd, get_auth
+from MgaRepo import layout
+from xml.etree import ElementTree
+import sys
+import os
+import re
+import time
+
+class VCSLogEntry(object):
+ def __init__(self, revision, author, date, lines=[], changed=[]):
+ self.revision = revision
+ self.author = author
+ self.date = date
+ self.changed = changed
+ self.lines = lines
+
+ def __lt__(self, other):
+ return (self.date < other.date)
+
+ def __eq__(self,other):
+ return (self.date == other.date)
+
+class VCS(object):
+ vcs_dirname = None
+ vcs_name = None
+ def __init__(self, path, url):
+ self.vcs_command = None
+ self.vcs_wrapper = "mga-ssh"
+ self.vcs_supports = {'clone' : False}
+ self.vcs_type = None
+ self.env_defaults = None
+ if not path and not url:
+ self._path = os.path.curdir
+ elif not path and url:
+ self._path = layout.package_name(layout.remove_current(url))
+ else:
+ self._path = path
+ self._url = url
+
+ def _execVcs(self, *args, **kwargs):
+ localcmds = ("add", "revert", "cleanup", "mv")
+ cmd = self.vcs_command + list(args)
+ kwargs["collecterr"] = kwargs.get("collecterr", False)
+ if kwargs.get("show"):
+ if not kwargs.get("local"):
+ kwargs["collecterr"] = True
+ else:
+ if self.vcs_command is "svn" and args[0] not in localcmds:
+ cmd.append("--non-interactive")
+ else:
+ if args[0] == "mv":
+ kwargs["collecterr"] = False
+ kwargs["cleanerr"] = kwargs.get("cleanerr", True)
+ if kwargs.get("xml"):
+ cmd.append("--xml")
+ try:
+ if args[0] in ('info', 'checkout','log'):
+ kwargs['info'] = True
+ else:
+ kwargs['info'] = False
+ return execcmd(*cmd, **kwargs)
+ except Error as e:
+ msg = None
+ if e.args:
+ if "Permission denied" in e.args:
+ msg = ("It seems ssh-agent or ForwardAgent are not setup "
+ "or your username is wrong. See "
+ "https://wiki.mageia.org/en/Packagers_ssh"
+ " for more information.")
+ elif "authorization failed" in e.args:
+ msg = ("Note that mgarepo does not support any HTTP "
+ "authenticated access.")
+ if kwargs.get("show") and \
+ not config.getbool("global", "verbose", 0):
+ # svn has already dumped error messages, we don't need to
+ # do it too
+ if msg:
+ sys.stderr.write("\n")
+ sys.stderr.write(msg)
+ sys.stderr.write("\n")
+ raise SilentError
+ elif msg:
+ raise Error("%s\n%s" % (e, msg))
+ raise
+
+ def _set_env(self):
+ wrapper = "mgarepo-ssh"
+ repsys = config.get("global", "mgarepo-cmd")
+ if repsys:
+ dir = os.path.dirname(repsys)
+ path = os.path.join(dir, wrapper)
+ if os.path.exists(path):
+ wrapper = path
+ defaults = {"SVN_SSH": wrapper}
+ os.environ.update(defaults)
+ raw = config.get("global", "svn-env")
+ if raw:
+ for line in raw.split("\n"):
+ env = line.strip()
+ if not env:
+ continue
+ try:
+ name, value = env.split("=", 1)
+ except ValueError:
+ sys.stderr.write("invalid svn environment line: %r\n" % env)
+ continue
+ os.environ[name] = value
+
+ def _execVcs_success(self, *args, **kwargs):
+ status, output = self._execVcs(*args, **kwargs)
+ return status == 0
+
+ def _add_log(self, cmd_args, received_kwargs, optional=0):
+ if (not optional or
+ "log" in received_kwargs or
+ "logfile" in received_kwargs):
+ ret = received_kwargs.get("log")
+ if ret is not None:
+ cmd_args.extend(("-m", ret))
+ ret = received_kwargs.get("logfile")
+ if ret is not None:
+ cmd_args.extend(("-F", ret))
+
+ def _add_revision(self, cmd_args, received_kwargs, optional=0):
+ if not optional or "rev" in received_kwargs:
+ ret = received_kwargs.get("rev")
+ if isinstance(ret, str):
+ if not ret.startswith("{"): # if not a datespec
+ try:
+ ret = int(ret)
+ except ValueError:
+ raise Error("invalid revision provided")
+ if ret:
+ cmd_args.extend(("-r", str(ret)))
+
+ def add(self, path, **kwargs):
+ cmd = ["add", path + '@' if '@' in path else path]
+ return self._execVcs_success(noauth=1, *cmd, **kwargs)
+
+ def copy(self, pathfrom, pathto, **kwargs):
+ cmd = ["copy", pathfrom + '@' if '@' in pathfrom else pathfrom, pathto + '@' if '@' in pathto else pathto]
+ self._add_revision(cmd, kwargs, optional=1)
+ self._add_log(cmd, kwargs)
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def remove(self, path, force=0, **kwargs):
+ cmd = ["remove", path + '@' if '@' in path else path]
+ self._add_log(cmd, kwargs)
+ if force:
+ cmd.append("--force")
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def mkdir(self, path, **kwargs):
+ cmd = ["mkdir", path + '@' if '@' in path else path]
+ if kwargs.get("parents"):
+ cmd.append("--parents")
+ self._add_log(cmd, kwargs)
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def _execVcs_commit(self, *cmd, **kwargs):
+ status, output = self._execVcs(*cmd, **kwargs)
+ match = re.search("Committed revision (?P<rev>\\d+)\\.$", output)
+ if match:
+ rawrev = match.group("rev")
+ return int(rawrev)
+
+ def commit(self, path, **kwargs):
+ cmd = ["commit", path + '@' if '@' in path else path]
+ if kwargs.get("nonrecursive"):
+ cmd.append("-N")
+ self._add_log(cmd, kwargs)
+ return self._execVcs_commit(*cmd, **kwargs)
+
+ def import_(self, path, url, **kwargs):
+ cmd = ["import", path, url]
+ self._add_log(cmd, kwargs)
+ return self._execVcs_commit(*cmd, **kwargs)
+
+ def export(self, url, targetpath, **kwargs):
+ cmd = ["export", url, targetpath]
+ self._add_revision(cmd, kwargs, optional=1)
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def checkout(self, url, targetpath, **kwargs):
+ cmd = ["checkout", url, targetpath]
+ self._add_revision(cmd, kwargs, optional=1)
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def clone(self, url, targetpath, **kwargs):
+ if self.vcs_supports['clone']:
+ cmd = ["clone", url, targetpath]
+ return self._execVcs_success(*cmd, **kwargs)
+ else:
+ raise Error("%s doesn't support 'clone'" % self.vcs_name)
+
+ def propget(self, propname, targets, **kwargs):
+ cmd = ["propget", propname, targets]
+ if kwargs.get("revprop"):
+ cmd.append("--revprop")
+ self._add_revision(cmd, kwargs)
+ status, output = self._execVcs(local=True, *cmd, **kwargs)
+ return output
+
+ def propset(self, propname, value, targets, **kwargs):
+ cmd = ["propset", propname, value, targets]
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def propedit(self, propname, target, **kwargs):
+ cmd = ["propedit", propname, target]
+ if kwargs.get("rev"):
+ cmd.append("--revprop")
+ self._add_revision(cmd, kwargs)
+ return self._execVcs_success(local=True, show=True, *cmd, **kwargs)
+
+ def revision(self, path, **kwargs):
+ cmd = ["info", path + '@' if '@' in path else path]
+ status, output = self._execVcs(local=True, *cmd, **kwargs)
+ if status == 0:
+ for line in output.splitlines():
+ if line.startswith("Last Changed Rev: "):
+ return int(line.split()[3])
+ return None
+
+ def info(self, path, **kwargs):
+ cmd = ["info", path + '@' if '@' in path else path]
+ status, output = self._execVcs(local=True, noerror=True, *cmd, **kwargs)
+ if (("Not a versioned resource" not in output) and ("svn: warning: W155010" not in output)):
+ return output.splitlines()
+ return None
+
+ def info2(self, *args, **kwargs):
+ lines = self.info(*args, **kwargs)
+ if lines is None:
+ return None
+ pairs = [[w.strip() for w in line.split(":", 1)] for line in lines]
+ info = {}
+ for pair in pairs:
+ if pair != ['']:
+ info[pair[0]]=pair[1]
+ return info
+
+ def ls(self, path, **kwargs):
+ cmd = ["ls", path + '@' if '@' in path else path]
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return output.split()
+ return None
+
+ def status(self, path, **kwargs):
+ cmd = ["status", path + '@' if '@' in path else path]
+ if kwargs.get("verbose"):
+ cmd.append("-v")
+ if kwargs.get("noignore"):
+ cmd.append("--no-ignore")
+ if kwargs.get("quiet"):
+ cmd.append("--quiet")
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return [(x[0], x[8:]) for x in output.splitlines()]
+ return None
+
+ def cleanup(self, path, **kwargs):
+ cmd = ["cleanup", path + '@' if '@' in path else path]
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def revert(self, path, **kwargs):
+ cmd = ["revert", path + '@' if '@' in path else path]
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return [x.split() for x in output.split()]
+ return None
+
+ def switch(self, url, oldurl=None, path=None, relocate=False, **kwargs):
+ cmd = ["switch"]
+ if relocate:
+ if oldurl is None:
+ raise Error("You must supply the old URL when "\
+ "relocating working copies")
+ cmd.append("--relocate")
+ cmd.append(oldurl)
+ cmd.append(url)
+ if path is not None:
+ cmd.append(path)
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def update(self, path, **kwargs):
+ cmd = ["update", path + '@' if '@' in path else path]
+ self._add_revision(cmd, kwargs, optional=1)
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return [x.split() for x in output.split()]
+ return None
+
+ def merge(self, url1, url2=None, rev1=None, rev2=None, path=None,
+ **kwargs):
+ cmd = ["merge"]
+ if rev1 and rev2 and not url2:
+ cmd.append("-r")
+ cmd.append("%s:%s" % (rev1, rev2))
+ cmd.append(url1)
+ else:
+ if not url2:
+ raise ValueError("url2 needed if two revisions are not provided")
+ if rev1:
+ cmd.append("%s@%s" % (url1, rev1))
+ else:
+ cmd.append(url1)
+ if rev2:
+ cmd.append("%s@%s" % (url2, rev2))
+ else:
+ cmd.append(url2)
+ if path:
+ cmd.append(path)
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return [x.split() for x in output.split()]
+ return None
+
+ def diff(self, pathurl1, pathurl2=None, **kwargs):
+ cmd = ["diff", pathurl1]
+ self._add_revision(cmd, kwargs, optional=1)
+ if pathurl2:
+ cmd.append(pathurl2)
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return output
+ return None
+
+ def cat(self, url, **kwargs):
+ cmd = ["cat", url]
+ self._add_revision(cmd, kwargs, optional=1)
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return output
+ return None
+
+ def log(self, url, start=None, end=0, limit=None, **kwargs):
+ cmd = ["log", "-v", url]
+ if start is not None or end != 0:
+ if start is not None and type(start) is not type(0):
+ try:
+ start = int(start)
+ except (ValueError, TypeError):
+ raise Error("invalid log start revision provided")
+ if type(end) is not type(0):
+ try:
+ end = int(end)
+ except (ValueError, TypeError):
+ raise Error("invalid log end revision provided")
+ start = start or "HEAD"
+ cmd.extend(("-r", "%s:%s" % (start, end)))
+ if limit is not None:
+ try:
+ limit = int(limit)
+ except (ValueError, TypeError):
+ raise Error("invalid limit number provided")
+ cmd.extend(("--limit", str(limit)))
+
+ status, output = self._execVcs(*cmd, xml=True, **kwargs)
+ if status != 0:
+ return None
+
+ xmllog = ElementTree.fromstring(output)
+ log = []
+ logentries = xmllog.getiterator("logentry")
+ for entry in logentries:
+ changed = []
+ lines = []
+ for pathelem in entry.getiterator("paths"):
+ path = pathelem.find("path")
+ from_rev = path.get("copyfrom-rev")
+ if from_rev:
+ from_rev = int(from_rev)
+ changed.append({"from_rev" : from_rev, "from_path" : path.get("copyfrom-path"), "action" : path.get("action"), "path" : path.text})
+ date = entry.findtext("date").split("T")
+ timestr = "%s %s" % (date[0], date[1].split(".")[0])
+ timetuple = time.strptime(timestr, "%Y-%m-%d %H:%M:%S")
+ lines.extend(entry.findtext("msg").rstrip().split("\n"))
+ logentry = VCSLogEntry(int(entry.attrib["revision"]),
+ entry.findtext("author"), timetuple, [line.rstrip() for line in lines], changed)
+ log.append(logentry)
+ log.sort()
+ log.reverse()
+ return log
+
+ def mv(self, path, dest, message=None, **kwargs):
+ cmd = ["mv", path, dest, ]
+ if message:
+ cmd.extend(("-m", str(message)))
+ else:
+ kwargs['show'] = True
+ self._add_log(cmd, kwargs)
+ return self._execVcs_success(*cmd, **kwargs)
+
+ def get_topdir(self):
+ vcsdir = os.path.join(self._path, self.vcs_dirname)
+ if os.path.exists(vcsdir) and os.path.isdir(vcsdir):
+ return self._path
+ else:
+ return None
+
+ @property
+ def path(self):
+ return self._path
+
+ @property
+ def url(self):
+ if not self._url:
+ self._url = self.info2(self._path)["URL"]
+ return self._url
+
+class VCSLook(object):
+ def __init__(self, repospath, txn=None, rev=None):
+ self.repospath = repospath
+ self.txn = txn
+ self.rev = rev
+ self.execcmd = None
+
+ def _execVcslook(self, cmd, *args, **kwargs):
+ execcmd_args = ["svnlook", cmd, self.repospath]
+ self._add_txnrev(execcmd_args, kwargs)
+ execcmd_args += args
+ execcmd_kwargs = {}
+ keywords = ["show", "noerror"]
+ for key in keywords:
+ if key in kwargs:
+ execcmd_kwargs[key] = kwargs[key]
+ return execcmd(*execcmd_args, **execcmd_kwargs)
+
+ def _add_txnrev(self, cmd_args, received_kwargs):
+ if "txn" in received_kwargs:
+ txn = received_kwargs.get("txn")
+ if txn is not None:
+ cmd_args.extend(("-t", txn))
+ elif self.txn is not None:
+ cmd_args.extend(("-t", self.txn))
+ if "rev" in received_kwargs:
+ rev = received_kwargs.get("rev")
+ if rev is not None:
+ cmd_args.exten(("-r", str(rev)))
+ elif self.rev is not None:
+ cmd_args.extend(("-r", str(self.rev)))
+
+ def changed(self, **kwargs):
+ status, output = self._execVcslook("changed", **kwargs)
+ if status != 0:
+ return None
+ changes = []
+ for line in output.splitlines():
+ line = line.rstrip()
+ if not line:
+ continue
+ entry = [None, None, None]
+ changedata, changeprop, path = None, None, None
+ if line[0] != "_":
+ changedata = line[0]
+ if line[1] != " ":
+ changeprop = line[1]
+ path = line[4:]
+ changes.append((changedata, changeprop, path))
+ return changes
+
+ def author(self, **kwargs):
+ status, output = self._execVcslook("author", **kwargs)
+ if status != 0:
+ return None
+ return output.strip()
+
+# vim:et:ts=4:sw=4
diff --git a/MgaRepo/binrepo.py b/MgaRepo/binrepo.py
index 89679b0..2d08300 100644
--- a/MgaRepo/binrepo.py
+++ b/MgaRepo/binrepo.py
@@ -102,12 +102,12 @@ def upload_binary(topdir, filename):
return
host = config.get("binrepo", "upload_host")
upload_bin_helper = get_helper("upload-bin")
- command = "ssh %s %s %s" % (host, upload_bin_helper, filename)
+ command = ["ssh", host, upload_bin_helper, filename]
try:
filein = open(filepath, 'r')
except Error as e:
raise Error("Could not open file %s\n" % filepath)
- status, output = execcmd(command, show=True, geterr=True, stdin=filein)
+ status, output = execcmd(command, show=True, collecterr=True, stdin=filein)
def import_binaries(topdir, pkgname):
"""Import all binaries from a given package checkout
diff --git a/MgaRepo/commands/buildrpm.py b/MgaRepo/commands/buildrpm.py
new file mode 100644
index 0000000..9caec45
--- /dev/null
+++ b/MgaRepo/commands/buildrpm.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python
+#
+from MgaRepo.command import do_command
+from MgaRepo.rpmutil import build_rpm
+from optparse import *
+
+HELP = """\
+Usage: mgarepo buildrpm [OPTIONS]
+
+Builds the binary RPM(s) (.rpm) file(s) of a given package.
+
+Options:
+ -bX Build stage option, where X is stage, default is -bb
+ -I Don't automatically try install missing build dependencies
+ -L Disable rpmlint check of packages built
+ -P USER Define the RPM packager information to USER
+ -d Use DNF
+ -q Quiet build output
+ -s Jump to specific build stage (--short-circuit)
+ -l Use subversion log to build rpm %changelog
+ -F Do not use full name & email for packagers in %changelog
+ -- Options and arguments following will be passed to rpmbuild
+
+"""
+
+def parse_options():
+ parser = OptionParser(HELP)
+ parser.add_option("-b", dest="build_cmd", default="a")
+ parser.add_option("-I", dest="installdeps", action="store_false", default=True)
+ parser.add_option("-L", dest="rpmlint", action="store_false", default=True)
+ parser.add_option("-P", dest="packager", default="")
+ parser.add_option("-d", "--dnf", dest="use_dnf", action="store_true", default=False)
+ parser.add_option("-q", "--quiet", dest="verbose", action="store_false", default=True)
+ parser.add_option("-s", "--short-circuit", dest="short_circuit", action="store_true", default=False)
+ parser.add_option("-l", dest="svnlog", action="store_true", default=False)
+ parser.add_option("-F", dest="fullnames", default=True,
+ action="store_false")
+ opts, args = parser.parse_args()
+ opts.rpmargs = parser.rargs
+ return opts
+
+def main():
+ do_command(parse_options, build_rpm)
+
+# vim:et:ts=4:sw=4
diff --git a/MgaRepo/commands/clone.py b/MgaRepo/commands/clone.py
new file mode 100644
index 0000000..86793c9
--- /dev/null
+++ b/MgaRepo/commands/clone.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+from MgaRepo import Error
+from MgaRepo.command import *
+from MgaRepo.rpmutil import clone
+import getopt
+import sys
+
+HELP = """\
+Usage: repsys co [OPTIONS] URL [LOCALPATH]
+
+Checkout the package source from the Mandriva repository.
+
+If the 'mirror' option is enabled, the package is obtained from the mirror
+repository.
+
+You can specify the distro branch to checkout from by using distro/pkgname.
+
+Options:
+ -d The distribution branch to checkout from
+ -b The package branch
+ -M Do not use the mirror (use the main repository)
+ -h Show this message
+ -F Do not convert svn usernames to full name & email
+
+Examples:
+ repsys clone pkgname
+ repsys clone -d 2009.0 pkgname
+ repsys clone 2009.0/pkgame
+ repsys clone http://repos/svn/cnc/snapshot/foo
+ repsys clone http://repos/svn/cnc/snapshot/foo foo-pkg
+"""
+
+def parse_options():
+ parser = OptionParser(help=HELP)
+ parser.add_option("--distribution", "-d", dest="distro", default=None)
+ parser.add_option("--branch", "-b", dest="branch", default=None)
+ parser.add_option("-F", dest="fullnames", default=True,
+ action="store_false")
+ opts, args = parser.parse_args()
+ if len(args) not in (1, 2):
+ raise Error("invalid arguments")
+ # here we don't use package_url in order to notify the user we are
+ # using the mirror
+ opts.pkgdirurl = args[0]
+ if len(args) == 2:
+ opts.path = args[1]
+ else:
+ opts.path = None
+ return opts
+
+def main():
+ do_command(parse_options, clone)
+
+# vim:et:ts=4:sw=4
diff --git a/MgaRepo/commands/getsrpm.py b/MgaRepo/commands/getsrpm.py
index 82bd626..5eb630b 100644
--- a/MgaRepo/commands/getsrpm.py
+++ b/MgaRepo/commands/getsrpm.py
@@ -30,6 +30,7 @@ Options:
-n Rename the package to include the revision number
-l Use subversion log to build rpm %changelog
-T FILE Template to be used to generate the %changelog
+ -F Do not use full name & email for packagers in %changelog
-M Do not use the mirror (use the main repository)
-h Show this message
--strict Check if the given revision contains changes in REPPKGURL
@@ -76,6 +77,8 @@ def parse_options():
parser.add_option("-n", dest="revname", action="store_true")
parser.add_option("-l", dest="svnlog", action="store_true")
parser.add_option("-T", dest="template", type="string", default=None)
+ parser.add_option("-F", dest="fullnames", default=True,
+ action="store_false")
parser.add_option("-M", "--no-mirror", action="callback",
callback=disable_mirror)
parser.add_option("--strict", dest="strict", default=False,
diff --git a/MgaRepo/commands/github.py b/MgaRepo/commands/github.py
new file mode 100644
index 0000000..dece050
--- /dev/null
+++ b/MgaRepo/commands/github.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+from MgaRepo import Error
+from MgaRepo.command import *
+from MgaRepo.GitHub import GitHub
+import getopt
+import sys
+
+HELP = """\
+Usage: mgarepo github [OPTIONS] URL
+
+Import a git-svn cloned repository to github
+
+Options:
+ -h Show this message
+
+Examples:
+ mgarepo github import existingpkg
+ mgarepo github import svn://svn.mageia.org/svn/packages/cauldron/existingpkg
+"""
+def github_clone(pkg, **kwargs):
+ github = GitHub()
+ github.clone_repository(pkg)
+
+def github_import(target=".", **kwargs):
+ github = GitHub()
+ github.import_package(target)
+
+def github_delete(pkg, **kwargs):
+ github = GitHub()
+ github.delete_repository(pkg)
+
+def parse_options():
+ parser = OptionParser(help=HELP)
+ opts, args = parser.parse_args()
+ if len(args) < 1:
+ raise Error("invalid arguments")
+ opts.func = globals().get("github_"+args[0], None)
+ if args[0] == "import":
+ if len(args) > 1:
+ opts.target = args[1]
+ elif args[0] == "delete" or args[0] == "clone":
+ opts.pkg = args[1]
+ else:
+ raise Error("invalid arguments: %s" % str(args))
+ return opts
+
+def dispatch_cmd(*args, **kwargs):
+ func = kwargs.pop("func", None)
+ if func:
+ func(**kwargs)
+ else:
+ raise Error("invalid command: %s %s" % (sys.argv[0], sys.argv[1]))
+
+def main():
+ do_command(parse_options, dispatch_cmd)
+
+# vim:et:ts=4:sw=4
diff --git a/MgaRepo/commands/log.py b/MgaRepo/commands/log.py
index 2181125..330a96a 100644
--- a/MgaRepo/commands/log.py
+++ b/MgaRepo/commands/log.py
@@ -6,6 +6,8 @@ from MgaRepo.rpmutil import sync
from MgaRepo.util import execcmd
import sys
import os
+import subprocess
+import shlex
HELP = """\
Usage: mgarepo log [OPTIONS] [PACKAGE]
@@ -57,9 +59,13 @@ def svn_log(pkgdirurl, verbose=False, limit=None, revision=None, releases=None):
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
- args.append("| less")
- rawcmd = " ".join(args)
- execcmd(rawcmd, show=True)
+ pager = shlex.split(os.environ.get("PAGER", "less"))
+ p = subprocess.Popen(args, stdout=subprocess.PIPE)
+ p2 = subprocess.Popen(pager, stdin=p.stdout)
+ p2.wait()
+ p.wait()
+ else:
+ execcmd(args, show=True)
def main():
do_command(parse_options, svn_log)
diff --git a/MgaRepo/commands/maintdb.py b/MgaRepo/commands/maintdb.py
index 9a97be8..2aa2a4c 100644
--- a/MgaRepo/commands/maintdb.py
+++ b/MgaRepo/commands/maintdb.py
@@ -33,8 +33,7 @@ def parse_options():
def maintdb(maintdb_args):
host = config.get("maintdb", "host", "maintdb.mageia.org")
maintdb_helper = get_helper("maintdb")
- cmd_args = ' '.join(maintdb_args)
- command = "ssh %s %s %s" % (host, maintdb_helper, cmd_args)
+ command = ["ssh", host, maintdb_helper] + maintdb_args
execcmd(command, show=True)
sys.exit(0)
diff --git a/MgaRepo/commands/rpmlog.py b/MgaRepo/commands/rpmlog.py
index 88dfc4b..28d66f1 100644
--- a/MgaRepo/commands/rpmlog.py
+++ b/MgaRepo/commands/rpmlog.py
@@ -25,6 +25,7 @@ Options:
-p Append changelog found in .spec file
-s Sort changelog entries, even from the old log
-M Do not use the mirror (use the main repository)
+ -F Do not use full name & email for packagers where available
-h Show this message
Examples:
@@ -45,13 +46,15 @@ def parse_options():
action="store_true")
parser.add_option("-M", "--no-mirror", action="callback",
callback=disable_mirror)
+ parser.add_option("-F", dest="fullnames", default=True,
+ action="store_false")
opts, args = parser.parse_args()
if len(args) != 1:
raise Error("invalid arguments")
opts.pkgdirurl = layout.package_url(args[0])
return opts
-def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort):
+def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort, fullnames):
another = None
if usespec:
svn = SVN()
@@ -59,7 +62,10 @@ def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort):
rawspec = svn.cat(specurl, rev=revision)
spec, another = split_spec_changelog(StringIO(rawspec))
newlog = get_changelog(pkgdirurl, another=another, rev=revision,
- size=size, sort=sort, template=template, oldlog=oldlog)
+ size=size, sort=sort, template=template, oldlog=oldlog, fullnames=fullnames)
+ # make sure stdout support unicode, otherwise it'll croak when encountered
+ if not "UTF-8" in sys.stdout.encoding:
+ sys.stdout = open(sys.stdout.fileno(), mode="w", encoding="UTF-8")
sys.stdout.writelines(newlog)
def main():
diff --git a/MgaRepo/commands/submit.py b/MgaRepo/commands/submit.py
index 9f05dca..d36290f 100644
--- a/MgaRepo/commands/submit.py
+++ b/MgaRepo/commands/submit.py
@@ -160,9 +160,10 @@ def list_targets(option, opt, val, parser):
raise Error("no submit host defined in mgarepo.conf")
createsrpm = get_helper("create-srpm")
#TODO make it configurable
- command = "ssh %s %s --list" % (host, createsrpm)
- execcmd(command, show=True)
- sys.exit(0)
+ args = ["ssh", host, createsrpm, "--list"]
+ execcmd(args, show=true)
+ sys.exit(0) # it is invoked via optparse callback, thus we need to
+ # force ending the script
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
if submithost is None:
@@ -197,8 +198,7 @@ def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
- command = subprocess.list2cmdline(cmdargs)
- status, output = execcmd(command)
+ status, output = execcmd(cmdargs)
if status == 0:
print("Package submitted!")
else:
diff --git a/MgaRepo/git.py b/MgaRepo/git.py
new file mode 100644
index 0000000..4e763f8
--- /dev/null
+++ b/MgaRepo/git.py
@@ -0,0 +1,192 @@
+from MgaRepo import Error, config
+from MgaRepo.util import execcmd
+from MgaRepo.VCS import *
+from MgaRepo.svn import SVN
+from MgaRepo.log import UserTagParser
+from os.path import basename, dirname, abspath, lexists, join
+from os import chdir, getcwd
+from tempfile import mkstemp
+import sys
+import re
+import time
+from xml.etree import ElementTree
+import subprocess
+
+class GITLogEntry(VCSLogEntry):
+ def __init__(self, revision, author, date):
+ VCSLogEntry.__init__(self, revision, author, data)
+
+class GIT(VCS):
+ vcs_dirname = ".git"
+ vcs_name = "git"
+ def __init__(self, path=None, url=None):
+ VCS.__init__(self, path, url)
+ self.vcs_command = config.get("global", "git-command", ["git"])
+ self.vcs_supports['clone'] = True
+ self.env_defaults = {"GIT_SSH": self.vcs_wrapper}
+
+ def configget(self, key="", location="--local"):
+ cmd = ["config", location, "--get-regexp", key]
+ config = None
+ status, output = self._execVcs(*cmd, noerror=True)
+ if not status and output:
+ config = eval("{'" + output.replace("\n", "',\n'").replace(" ", "' : '") + "'}")
+ return config
+
+ def configset(self, config, location="--local"):
+ cmd = ("config", location)
+ for pair in config.items():
+ status, output = self._execVcs(*cmd + pair)
+ if status:
+ return False
+ return True
+
+ def clone(self, url, targetpath, fullnames=True, **kwargs):
+ for vcs in (SVN, GIT):
+ if lexists(join(targetpath, vcs.vcs_dirname)):
+ raise Error("target path %s already contains %s repository, aborting..." % (targetpath, vcs.vcs_name))
+
+ self.init(url, targetpath, fullnames=True, **kwargs)
+ if url.split(':')[0].find("svn") < 0:
+ return VCS.clone(self, url, **kwargs)
+ else:
+ return self.update(targetpath, clone=True, **kwargs)
+
+ def init(self, url, targetpath, fullnames=True, branch=None, **kwargs):
+ # verify repo url
+ execcmd("svn", "info", url)
+
+ topurl = dirname(url)
+ trunk = basename(url)
+ tags = "releases"
+ # cloning svn braches as well should rather be optionalif reenabled..
+ #cmd = ["svn", "init", topurl, "--trunk="+trunk, "--tags="+tags", targetpath]
+
+ cmd = ["svn", "init", url, abspath(targetpath)]
+ self._execVcs(*cmd, **kwargs)
+ os.environ.update({"GIT_WORK_TREE" : abspath(targetpath), "GIT_DIR" : join(abspath(targetpath),".git")})
+
+ if fullnames:
+ usermap = UserTagParser()
+ # store configuration in local git config so that'll be reused later when ie. updating
+ gitconfig = {"svn-remote.authorlog.url" : usermap.url,
+ "svn-remote.authorlog.defaultmail": usermap.defaultmail}
+ self.configset(gitconfig)
+
+ if branch:
+ execcmd(("git", "init", "-q", self.path), **kwargs)
+ execcmd(("git", "checkout", "-q", branch), **kwargs)
+ cmd = ["svn", "rebase", "--local"]
+ status, output = self._execVcs(*cmd, **kwargs)
+
+ return True
+
+ def info(self, path, **kwargs):
+ cmd = ["svn", "info", path + '@' if '@' in path else path]
+ status, output = self._execVcs(local=True, noerror=True, *cmd, **kwargs)
+ if (("Not a git repository" not in output) and \
+ ("Unable to determine upstream SVN information from working tree history" not in output)):
+ return output.splitlines()
+ return None
+
+ def status(self, path, **kwargs):
+ cmd = ["status", path + '@' if '@' in path else path]
+ if kwargs.get("verbose"):
+ cmd.append("-v")
+ if kwargs.get("noignore"):
+ cmd.append("-u")
+ if kwargs.get("quiet"):
+ cmd.append("-s")
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return [(x[0], x[8:]) for x in output.splitlines()]
+ return None
+
+ def update(self, targetpath, clone=False, **kwargs):
+ os.environ.update({"GIT_WORK_TREE" : abspath(targetpath), "GIT_DIR" : join(abspath(targetpath),".git")})
+
+ if not clone:
+ cmd = ["svn", "log", "--oneline", "--limit=1"]
+ retval, result = self._execVcs(*cmd)
+ if retval:
+ return retval
+
+ revision = result.split()
+
+ if revision[0][0] == 'r':
+ startrev = "-r"+str(int(revision[0][1:])+1)
+ else:
+ startrev = "BASE"
+
+ cmd = ["svn", "propget", "svn:entry:committed-rev"]
+ retval, lastrev = self._execVcs(*cmd)
+ if retval:
+ return retval
+
+ #cmd = ["config", "--get-regexp", '^svn-remote.svn.(url|fetch)']
+ cmd = ["config", "--get", "svn-remote.svn.url"]
+ retval, result = self._execVcs(*cmd)
+ if retval:
+ return retval
+
+ #result = result.strip().split()
+ #url = result[1] + "/" + result[3].split(":")[0]
+ url = result.strip()
+
+ # To speed things up on huge repositories, we'll just grab all the
+ # revision numbers for this specific directory and grab these only
+ # in stead of having to go through each and every revision...
+ cmd = ["svn", "log", "-g", "--xml", url]
+ if not clone:
+ cmd.append("%s:%s" % (startrev,lastrev))
+ retval, result = execcmd(*cmd)
+ if retval:
+ return retval
+
+ xmllog = ElementTree.fromstring(result)
+ logentries = xmllog.getiterator("logentry")
+ revisions = []
+ for entry in logentries:
+ revisions.append(int(entry.attrib["revision"]))
+ revisions.sort()
+
+ fetchcmd = ["svn", "fetch", "--log-window-size=1000"]
+ gitconfig = self.configget("svn-remote.authorlog")
+ if gitconfig:
+ usermap = UserTagParser(url=gitconfig.get("svn-remote.authorlog.url"),defaultmail=gitconfig.get("svn-remote.authorlog.defaultmail"))
+ usermapfile = usermap.get_user_map_file()
+ fetchcmd.extend(("--authors-file", usermapfile))
+ fetchcmd.append("")
+
+ while revisions:
+ fetchcmd[-1] = "-r%d"%revisions.pop(0)
+ self._execVcs(*fetchcmd, **kwargs)
+ if gitconfig:
+ usermap.cleanup()
+
+ cmd = ["svn", "rebase", "--log-window-size=1000", "--local", "--fetch-all", "git-svn"]
+ status, output = self._execVcs(*cmd, **kwargs)
+ if status == 0:
+ return [x.split() for x in output.split()]
+ return None
+
+ def remote(self, *args, **kwargs):
+ cmd = ["remote"] + list(args)
+ status, output = self._execVcs(*cmd, **kwargs)
+ return status, output
+
+ def pull(self, *args, **kwargs):
+ cmd = ["pull"] + list(args)
+ status, output = self._execVcs(*cmd, **kwargs)
+ return status, output
+
+ def push(self, *args, **kwargs):
+ cmd = ["push"] + list(args)
+ status, output = self._execVcs(*cmd, **kwargs)
+ return status, output
+
+class GITLook(VCSLook):
+ def __init__(self, repospath, txn=None, rev=None):
+ VCSLook.__init__(self, repospath, txn, rev)
+
+# vim:et:ts=4:sw=4
diff --git a/MgaRepo/log.py b/MgaRepo/log.py
index fab7d6e..f6c0fc9 100644
--- a/MgaRepo/log.py
+++ b/MgaRepo/log.py
@@ -8,6 +8,7 @@ from io import StringIO
import sys
import os
+import os.path
import re
import time
import locale
@@ -27,7 +28,6 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False):
Is here where things should be changed if "automatic release increasing"
will be used.
"""
- from MgaRepo.rpmutil import rpm_macros_defs
svn = SVN()
pkgcurrenturl = os.path.join(pkgdirurl, "current")
specurl = os.path.join(pkgcurrenturl, "SPECS")
@@ -43,17 +43,11 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False):
if not found:
raise Error("no .spec file found inside %s" % specurl)
specpath = found[0]
- options = rpm_macros_defs(macros)
- command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' "
- "--specfile %s %s") %
- (specpath, options))
- pipe = subprocess.Popen(command, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, shell=True)
- pipe.wait()
- output = pipe.stdout.read().decode('utf8')
- error = pipe.stderr.read().decode('utf8')
- if pipe.returncode != 0:
- raise Error("Error in command %s: %s" % (command, error))
+ options = [("--define", expr) for expr in macros]
+ command = ["rpm", "-q", "--qf", "%{EPOCH}:%{VERSION}-%{RELEASE}\n",
+ "--specfile", specpath]
+ command.extend(options)
+ status, output = execcmd(*command)
releases = output.split()
try:
epoch, vr = releases[0].split(":", 1)
@@ -81,17 +75,11 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False):
if not found:
raise Error("no .src.rpm file found inside %s" % srpmurl)
srpmpath = found[0]
- options = rpm_macros_defs(macros)
- command = (("rpm -qp --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' "
- " %s %s") %
- (srpmpath, options))
- pipe = subprocess.Popen(command, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, shell=True)
- pipe.wait()
- output = pipe.stdout.read().decode('utf8')
- error = pipe.stderr.read().decode('utf8')
- if pipe.returncode != 0:
- raise Error("Error in command %s: %s" % (command, error))
+ options = [("--define", expr) for expr in macros]
+ command = ["rpm", "-q", "--qf", "%{EPOCH}:%{VERSION}-%{RELEASE}\n",
+ "--specfile", specpath]
+ command.extend(options)
+ status, output = execcmd(*command)
releases = output.split()
try:
epoch, vr = releases[0].split(":", 1)
@@ -257,11 +245,12 @@ def group_revisions_by_author(currentlog):
emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>")
-
+usermap = {}
def get_author_name(author):
found = emailpat.match(config.get("users", author, author))
- name = ((found and found.group("name")) or author)
- email = ((found and found.group("email")) or author)
+ gold = emailpat.match(usermap.get(author,""))
+ name = ((found and found.group("name")) or (gold and gold.group("name")) or author)
+ email = ((found and found.group("email")) or (gold and gold.group("email")) or author+"@mageia.org")
return name, email
def parse_raw_date(rawdate):
@@ -354,15 +343,18 @@ def dump_file(releases, currentlog=None, template=None):
first = False
else:
draft = draft + spaces + line + "\n"
- draft += '\n'
+ if rel is not releases_author[-1]:
+ draft += "\n"
else:
# default template
if not releases_author[-1].visible:
releases_author = releases_author[:-1]
for rel in releases_author:
if not rel.released:
- draft = " (not released yet)\n"
- draft = draft + "* {0} {1} <{2}> {3}-{4}\n+ Revision: {5}\n".format(rel.date, rel.author_name, rel.author_email, rel.version, rel.release, rel.revision)
+ unreleased = " (not released yet)\n"
+ else:
+ unreleased = ""
+ draft = draft + "* {0} {1} <{2}> {3}-{4}\n{5}+ Revision: {6}\n".format(rel.date, rel.author_name, rel.author_email, rel.version, rel.release, unreleased, rel.revision)
if not rel.visible:
draft = draft + "+ rebuild (emptylog)\n"
for rev in rel.release_revisions:
@@ -375,7 +367,8 @@ def dump_file(releases, currentlog=None, template=None):
for rev in author.revisions:
for line in rev.lines:
draft = draft + line + "\n"
- draft += "\n"
+ if rel is not releases_author[-1]:
+ draft += "\n"
return draft
class InvalidEntryError(Exception):
@@ -606,9 +599,74 @@ def get_old_log(pkgdirurl):
chlog.seek(0)
return chlog
+from html.parser import HTMLParser
+from urllib.request import urlopen
+class UserTagParser(HTMLParser):
+ li = False
+ ahref = False
+ userpage = None
+ namepat = re.compile("(?P<name>.*?)\s*\((?P<user>.*?)\)")
+ usermap = {}
+ usermapfile = None
+
+ def __init__(self, url=None, defaultmail=None, *cmd, **kwargs):
+ HTMLParser.__init__(self, *cmd, **kwargs)
+ self.url = url or "http://people.mageia.org/u/"
+ self.defaultmail = defaultmail or "mageia.org"
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "li":
+ self.li = True
+ if self.li and tag == "a":
+ for att in attrs:
+ if att[0] == "href":
+ self.ahref = True
+ self.userpage = att[1]
+
+ def handle_endtag(self, tag):
+ if self.li and tag == "a":
+ self.ahref = False
+ self.userpage = None
+ if tag == "li":
+ self.li = False
+
+ def handle_data(self, data):
+ if self.li and self.ahref:
+ found = self.namepat.match(data)
+ if found:
+ user = found.group("user")
+ name = found.group("name")
+ if user and name and user+".html" == self.userpage:
+ self.usermap[user] = "%s <%s@%s>" % (name, user, self.defaultmail)
+
+ def get_user_map(self):
+ f = urlopen(self.url)
+ userhtml = f.read().decode("UTF-8")
+ f.close()
+ self.feed(userhtml)
+ return self.usermap
+
+ def get_user_map_file(self):
+ if not self.usermap:
+ self.get_user_map()
+ self.usermapfile = tempfile.mkstemp(suffix=".txt", prefix="usermap")
+ f = open(self.usermapfile[0], "w", encoding="UTF-8")
+ f.writelines("%s = %s\n" % user for user in sorted(self.usermap.items()))
+ f.close()
+ return self.usermapfile[1]
+
+ def cleanup(self):
+ if os.path.exists(self.usermapfile[1]):
+ os.unlink(self.usermapfile[1])
+
+def _map_user_names():
+ if not usermap:
+ parser = UserTagParser()
+ usermap.update(parser.get_user_map())
+
def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
submit=False, sort=False, template=None, macros=[], exported=None,
- oldlog=False, create=False):
+ oldlog=False, create=False, fullnames=False):
"""Generates the changelog for a given package URL
@another: a stream with the contents of a changelog to be merged with
@@ -634,6 +692,9 @@ def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
"""
newlog = StringIO()
if svn:
+ if fullnames:
+ if not usermap:
+ _map_user_names()
rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
template=template, macros=macros, exported=exported, create=create)
newlog.write(rawsvnlog)
@@ -648,19 +709,21 @@ def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
return newlog
def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
- submit=False, sort=False, template=None, macros=[], exported=None, create=False):
+ submit=False, sort=False, template=None, macros=[], exported=None, create=False, fullnames=False):
with open(specfile, encoding = 'utf-8') as fi:
spec, oldchlog = split_spec_changelog(fi)
another = None
if config.getbool("log", "merge-spec", False):
another = oldchlog
sort = sort or config.getbool("log", "sort", False)
+ if fullnames:
+ _map_user_names()
chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size,
submit=submit, sort=sort, template=template, macros=macros,
exported=exported, oldlog=True, create=create)
with open(specfile, "w", encoding='utf-8') as fo:
fo.writelines(spec)
- fo.write("\n\n%changelog\n")
+ fo.write("\n%changelog\n")
fo.writelines(chlog)
if __name__ == "__main__":
diff --git a/MgaRepo/rpmutil.py b/MgaRepo/rpmutil.py
index 769167c..49ef07f 100644
--- a/MgaRepo/rpmutil.py
+++ b/MgaRepo/rpmutil.py
@@ -1,9 +1,10 @@
#!/usr/bin/python3
from MgaRepo import Error, config
from MgaRepo import mirror, layout, log, binrepo
+from MgaRepo.git import GIT
from MgaRepo.svn import SVN
from MgaRepo.simplerpm import SRPM
-from MgaRepo.util import execcmd
+from MgaRepo.util import execcmd, CommandError
from MgaRepo.command import default_parent
import rpm
import urllib.parse
@@ -14,13 +15,38 @@ import glob
import sys
import os
+def detectVCS(url):
+ if ':' in url:
+ protocol,uri = url.split(":")
+ if "svn" in protocol:
+ return SVN(url=url)
+ elif "git" in protocol:
+ return GIT(url=url)
+ elif "http" in protocol:
+ if uri.endswith(".git"):
+ return GIT(url=url)
+ elif "svn" in uri:
+ return SVN(url=url)
+ raise Error("Unknown protocol %s for %s" % (protocol, url))
+ elif os.path.exists(url) and os.path.isdir(url):
+ while True:
+ url = os.path.abspath(url)
+ for vcs in (SVN, GIT):
+ vcsdir = os.path.join(url, vcs.vcs_dirname)
+ if os.path.exists(vcsdir) and os.path.isdir(vcsdir):
+ return vcs(path=url)
+ url = os.path.dirname(url)
+ if url == "/":
+ break
+ raise Error("No supported repository found at path: %s" % url)
+
def get_spec(pkgdirurl, targetdir=".", submit=False):
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
tmpdir = tempfile.mktemp()
try:
geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
mirror.info(geturl)
- svn.export("'%s'" % geturl, tmpdir)
+ svn.export(geturl, tmpdir)
speclist = glob.glob(os.path.join(tmpdir, "*.spec"))
if not speclist:
raise Error("no spec files found")
@@ -33,14 +59,9 @@ def get_spec(pkgdirurl, targetdir=".", submit=False):
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
-def rpm_macros_defs(macros):
- defs = ("--define \"%s %s\"" % macro for macro in macros)
- args = " ".join(defs)
- return args
-
#FIXME move it to another module
def rev_touched_url(url, rev):
- svn = SVN()
+ svn = detectVCS(url)
info = svn.info2(url)
if info is None:
raise Error("can't fetch svn info about the URL: %s" % url)
@@ -95,16 +116,21 @@ def get_srpm(pkgdirurl,
template = None,
macros = [],
verbose = 0,
- strict = False):
- svn = SVN()
+ strict = False,
+ fullnames = False):
+ svn = detectVCS(pkgdirurl)
tmpdir = tempfile.mktemp()
- topdir = "--define '_topdir %s'" % tmpdir
- builddir = "--define '_builddir %s/%s'" % (tmpdir, "BUILD")
- rpmdir = "--define '_rpmdir %s/%s'" % (tmpdir, "RPMS")
- sourcedir = "--define '_sourcedir %s/%s'" % (tmpdir, "SOURCES")
- specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS")
- srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS")
- patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES")
+ topdir = "_topdir %s" % tmpdir
+ builddir = "_builddir %s/%s" % (tmpdir, "BUILD")
+ rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS")
+ sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES")
+ specdir = "_specdir %s/%s" % (tmpdir, "SPECS")
+ srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS")
+ patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES")
+ temppath = "_tmppath %s" % (tmpdir)
+
+ rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir,
+ sourcedir, specdir, srcrpmdir, patchdir, temppath)]
try:
if mode == "version":
@@ -133,24 +159,29 @@ def get_srpm(pkgdirurl,
if config.getbool("srpm", "run-prep", False):
makefile = os.path.join(tmpdir, "Makefile")
if os.path.exists(makefile):
- execcmd("make", "-C", tmpdir, "srpm-prep")
+ execcmd(("make", "-C", tmpdir, "srpm-prep"))
if not speclist:
raise Error("no spec files found")
spec = speclist[0]
- defs = rpm_macros_defs(macros)
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
if packager:
packager = " --define 'packager %s'" % packager
+ sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
+ args = [sourcecmd, "-bs", "--nodeps"]
+ for pair in rpmdefs:
+ args.extend(pair)
+ for pair in macros:
+ args.extend(("--define", "%s %s" % pair))
+ args.append(spec)
if svnlog:
submit = not not revision
try:
log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
- template=template, macros=macros, exported=tmpdir)
+ template=template, macros=macros, exported=tmpdir, fullnames=fullnames)
except:
- execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" %
- (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir,
- srcrpmdir, patchdir, packager, spec, defs))
+ #cmd = [sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir
+ execcmd(args)
cp_srpms(revision, revname, geturl, targetdirs, srpmsdir, verbose)
log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
template=template, macros=macros, exported=tmpdir, create=True)
@@ -162,9 +193,15 @@ def get_srpm(pkgdirurl,
if status != 0:
raise Error("script %s failed" % script)
- execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" %
- (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir,
- srcrpmdir, patchdir, packager, spec, defs))
+ try:
+ execcmd(args)
+ except CommandError as e:
+ if config.getbool("global", "verbose"):
+ cmdline = e.cmdline + "\n"
+ else:
+ cmdline = ""
+ raise Error("error while creating the source RPM "
+ "(with %s):\n%s%s" % (sourcecmd, cmdline, e.output))
# copy the generated SRPMs to their target locations
targetsrpms = cp_srpms(revision, revname, geturl, targetdirs, srpmsdir, verbose)
@@ -175,7 +212,7 @@ def get_srpm(pkgdirurl,
def patch_spec(pkgdirurl, patchfile, log=""):
#FIXME use get_spec
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
tmpdir = tempfile.mktemp()
try:
geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
@@ -184,7 +221,7 @@ def patch_spec(pkgdirurl, patchfile, log=""):
if not speclist:
raise Error("no spec files found")
spec = speclist[0]
- status, output = execcmd("patch", spec, patchfile)
+ status, output = execcmd(["patch", spec, patchfile])
if status != 0:
raise Error("can't apply patch:\n%s\n" % output)
else:
@@ -195,7 +232,6 @@ def patch_spec(pkgdirurl, patchfile, log=""):
def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
baseurl=None, baseold=None, logmsg=None, rename=True):
- svn = SVN()
srpm = SRPM(srpmfile)
tmpdir = tempfile.mktemp()
if baseurl:
@@ -203,6 +239,7 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
else:
pkgurl = layout.package_url(srpm.name, distro=branch,
mirrored=False)
+ svn = detectVCS(pkgdirurl)
print("Importing package to %s" % pkgurl)
try:
if srpm.epoch:
@@ -264,9 +301,9 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
svn.remove(entrypath)
# Copy all files
- execcmd("cp -rlf", uspecsdir, currentdir)
+ execcmd(["cp", "-rf", uspecsdir, currentdir])
if os.path.isdir(usourcesdir):
- execcmd("cp -rlf", usourcesdir, currentdir)
+ execcmd(["cp", "-rlf", usourcesdir, currentdir])
# Add new entries
for entry in [x for x in uspecsentries
@@ -355,8 +392,106 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
log="Copying release %s-%s to releases/ directory." %
(version, srpm.release))
+def build_rpm(build_cmd="b",
+ verbose=True,
+ rpmlint=True,
+ short_circuit=False,
+ packager = None,
+ installdeps = True,
+ use_dnf = False,
+ svnlog = False,
+ fullnames = True,
+ macros = [],
+ **rpmargs):
+ top = os.getcwd()
+ topdir = "_topdir %s" % top
+ builddir = "_builddir %s/%s" % (top, "BUILD")
+ rpmdir = "_rpmdir %s/%s" % (top, "RPMS")
+ sourcedir = "_sourcedir %s/%s" % (top, "SOURCES")
+ specdir = "_specdir %s/%s" % (top, "SPECS")
+ srcrpmdir = "_srcrpmdir %s/%s" % (top, "SRPMS")
+ patchdir = "_patchdir %s/%s" % (top, "SOURCES")
+
+ build = os.path.join(top, "BUILD")
+ if not os.path.exists(build):
+ os.mkdir(build)
+ specsdir = os.path.join(top, "SPECS")
+ speclist = glob.glob(os.path.join(specsdir, "*.spec"))
+ if not speclist:
+ raise Error("no spec files found")
+ spec = speclist[0]
+
+ # If we're building package with %changelog, we'll make a temporary
+ # copy of the spec file with %changelog applied that we'll use for
+ # building. This way we avoid modifying files in repository.
+ # TODO: implement support for external changelog in rpm
+ if svnlog:
+ vcs = detectVCS(top)
+ specsdir = tempfile.mkdtemp()
+ shutil.copy(spec, specsdir)
+ specdir = "_specdir "+specsdir
+ spec = os.path.join(specsdir,os.path.basename(spec))
+ info = vcs.info2(top)
+ pkgdirurl = layout.remove_current(info["URL"])
+ log.specfile_svn2rpm(pkgdirurl, spec, rev=None, submit=False,
+ template=None, macros=macros, exported=top, fullnames=fullnames)
+
+ rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir,
+ sourcedir, specdir, srcrpmdir, patchdir)]
+
+ if packager:
+ rpmdefs.append(("--define", "packager %s" % packager))
+
+ if rpmlint:
+ rpmdefs.append(("--define", "_build_pkgcheck_set %{_bindir}/rpmlint"))
+
+ rpmbuild = config.get("helper", "rpmbuild", "rpmbuild")
+ args = [rpmbuild, spec]
+ if short_circuit:
+ args.append("--short-circuit")
+ for pair in rpmdefs:
+ args.extend(pair)
+ for pair in macros:
+ args.extend(("--define", "%s %s" % pair))
+ args.extend(("--define", "_disable_source_fetch 0"))
+ args.extend(*rpmargs.values())
+ os.environ["LC_ALL"] = "C"
+ # First check whether dependencies are satisfied
+ status, output = execcmd(*args + ["--nobuild"], show=verbose, collecterr=True, noerror=True)
+ if status:
+ if "error: Failed build dependencies:" in output:
+ if not installdeps:
+ raise Error("Automatic installation of dependencies disabled,"
+ "aborting...")
+ else:
+ if verbose:
+ print("Installing missing build dependencies")
+ if use_dnf:
+ pkg_mgr_base = ["dnf"]
+ pkg_mgr_builddep = pkg_mgr_base + ["--assume-yes", "--setopt=install_weak_deps=False", "builddep"]
+ else:
+ pkg_mgr_base = ["urpmi"]
+ pkg_mgr_builddep = pkg_mgr_base + ["--auto", "--buildrequires", "--no-recommends"]
+ if os.getuid() != 0:
+ print("Trying to obtain privileges for installing build dependencies:")
+ sudocheck = ["sudo", "-l"] + pkg_mgr_base
+ status, output = execcmd(*sudocheck, collecter=True, noerror=True)
+ if status:
+ raise Error("%s\nFailed! Cannot proceed without, aborting..."
+ % output.splitlines()[-1])
+ cmd_base = ["sudo"] + pkg_mgr_builddep
+ else:
+ cmd_base = pkg_mgr_builddep
+ cmd = cmd_base + [spec]
+ status, output = execcmd(*cmd, show=verbose, collecter=True, noerror=True)
+
+ status, output = execcmd(*args + ["-b"+build_cmd], show=verbose)
+ if svnlog:
+ if os.path.isdir(specsdir):
+ shutil.rmtree(specsdir)
+
def create_package(pkgdirurl, log="", verbose=0):
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
tmpdir = tempfile.mktemp()
try:
basename = layout.package_name(pkgdirurl)
@@ -397,7 +532,7 @@ revision: %s
return log
def mark_release(pkgdirurl, version, release, revision):
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
releasesurl = layout.checkout_url(pkgdirurl, releases=True)
versionurl = "/".join([releasesurl, version])
releaseurl = "/".join([versionurl, release])
@@ -419,7 +554,7 @@ def mark_release(pkgdirurl, version, release, revision):
log=markreleaselog)
def check_changed(pkgdirurl, all=0, show=0, verbose=0):
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
if all:
baseurl = pkgdirurl
packages = []
@@ -490,33 +625,48 @@ def checkout(pkgdirurl, path=None, revision=None, branch=None, distro=None, back
if path is None:
path = layout.package_name(pkgdirurl)
mirror.info(current, write=True)
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
svn.checkout(current, path, rev=revision, show=1)
if not spec:
binrepo.download_binaries(path)
-def getpkgtopdir(basedir=None):
- #FIXME this implementation doesn't work well with relative path names,
- # which is something we need in order to have a friendlier output
- if basedir is None:
- basedir = os.path.curdir
- while not ispkgtopdir(basedir):
- if os.path.abspath(basedir) == "/":
- raise Error("can't find top package directories SOURCES and SPECS")
- basedir = os.path.join(basedir, os.path.pardir)
- if basedir.startswith("./"):
- basedir = basedir[2:]
- return basedir
-
-def ispkgtopdir(path=None):
+def clone(pkgdirurl, path=None, revision=None, branch=None, distro=None, backports=None,
+ spec=False, fullnames = True, bindownload = True):
+ o_pkgdirurl = pkgdirurl
+ pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro, backports=backports)
+ append = None
+ if spec:
+ append = "SPECS"
+ current = layout.checkout_url(pkgdirurl, branch=branch, backports=backports,
+ append_path=append)
+ if path is None:
+ path = layout.package_name(pkgdirurl)
+ mirror.info(current, write=True)
+ git = GIT()
+ git.clone(current, path, fullnames=fullnames, show=1)
+ if not spec and bindownload:
+ binrepo.download_binaries(path)
+
+def getpkgtopdir(basedir=os.path.curdir):
+ vcs = detectVCS(basedir)
+ if vcs:
+ basedir = os.path.relpath(vcs.get_topdir())
+ if ispkgtopdir(basedir, vcs_dirname=vcs.vcs_dirname):
+ return basedir
+ raise Error("can't find top package directories SOURCES and SPECS")
+
+def ispkgtopdir(path=None, vcs_dirname=None):
if path is None:
path = os.getcwd()
names = os.listdir(path)
- return (".svn" in names and "SPECS" in names and "SOURCES" in names)
+ if not vcs_dirname:
+ vcs = detectVCS(path)
+ vcs_dirname = vcs.vcs_dirname
+ return (vcs_dirname in names and "SPECS" in names and "SOURCES" in names)
def sync(dryrun=False, commit=False, download=False):
- svn = SVN()
topdir = getpkgtopdir()
+ svn = detectVCS(topdir)
spath = binrepo.sources_path(topdir)
binrepoentries = binrepo.parse_sources(spath)
# run svn info because svn st does not complain when topdir is not an
@@ -598,7 +748,7 @@ def sync(dryrun=False, commit=False, download=False):
upload([path], commit=commit)
def commit(target=".", message=None, logfile=None):
- svn = SVN()
+ svn = detectVCS(target)
status = svn.status(target, quiet=True)
if not status:
print("nothing to commit")
@@ -633,39 +783,41 @@ def spec_sources(topdir):
return sources
def update(target=None):
- svn = SVN()
+ vcs = None
info = None
- svn_target = None
+ vcs_target = None
br_target = None
if target:
- svn_target = target
+ vcs_target = target
else:
top = getpkgtopdir()
- svn_target = top
+ vcs_target = top
br_target = top
- if svn_target:
- svn.update(svn_target, show=True)
+ if vcs_target:
+ vcs = detectVCS(vcs_target)
+ vcs.update(vcs_target, show=True)
if br_target:
- info = svn.info2(svn_target)
- if not br_target and not svn_target:
- raise Error("target not in SVN nor in binaries "\
- "repository: %s" % target)
+ if not vcs:
+ vcs = detectVCS(br_target)
+ info = vcs.info2(vcs_target)
+ if not br_target and not vcs_target:
+ raise Error("target not in %s nor in binaries "\
+ "repository: %s" % (type(vcs).__name__,target))
url = info["URL"]
binrepo.download_binaries(br_target)
def upload(paths, commit=False):
+ topdir = getpkgtopdir()
+ svn = detectVCS(topdir)
for path in paths:
if os.path.isdir(path) or binrepo.is_binary(path):
- topdir = getpkgtopdir()
binrepo.upload_binary(topdir, os.path.basename(path))
binrepo.update_sources(topdir, added=[path])
if commit:
- svn = SVN()
silent = config.get("log", "ignore-string", "SILENT")
message = "%s: new file %s" % (silent, path)
svn.commit(binrepo.sources_path(topdir), log=message)
else:
- svn = SVN()
svn.add(path, local=True)
if commit:
silent = config.get("log", "ignore-string", "SILENT")
@@ -674,16 +826,15 @@ def upload(paths, commit=False):
def delete(paths, commit=False):
silent = config.get("log", "ignore-string", "SILENT")
+ topdir = getpkgtopdir()
+ svn = detectVCS(topdir)
for path in paths:
message = "%s: delete file %s" % (silent, path)
if binrepo.is_binary(path):
- topdir = getpkgtopdir()
binrepo.update_sources(topdir, removed=[os.path.basename(path)])
if commit:
- svn = SVN()
svn.commit(binrepo.sources_path(topdir), log=message)
else:
- svn = SVN()
svn.remove(path, local=True)
if commit:
svn.commit(path, log=message)
@@ -692,7 +843,7 @@ def obsolete(pkgdirurl, branch=None, distro=None, backports=None, commit=False,
o_pkgdirurl = pkgdirurl
pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro, backports=backports)
pkgdest = layout.package_url(o_pkgdirurl, obsolete=True, backports=backports)
- svn = SVN()
+ svn = detectVCS(pkgdirurl)
svn.mv(pkgdirurl, pkgdest, message=log)
if commit:
svn.commit(path, log=log)
@@ -727,7 +878,7 @@ def get_submit_info(path):
if not os.path.isdir(os.path.join(path, ".svn")):
raise Error("subversion directory not found")
- svn = SVN()
+ svn = detectVCS(path)
# Now, extract the package name.
info = svn.info2(path)
@@ -766,4 +917,25 @@ def get_submit_info(path):
return name, url, max
+def get_pkg_tag(tag, path=os.path.curdir, subpkg=None):
+ topdir = getpkgtopdir(path)
+ speclist = glob.glob(os.path.join(topdir, "SPECS", "*.spec"))
+ if not speclist:
+ raise Error("no spec files found")
+ specfile = speclist[0]
+
+ pkg = rpm.spec(specfile)
+ if subpkg is None:
+ header = pkg.sourceHeader
+ elif isinstance(subpkg,int):
+ header = pkg.packages(subpkg)
+ else:
+ raise Error("Subpkg must be the index number of a package,"\
+ "or None for source package")
+
+ if isinstance(header[tag],bytes):
+ return header[tag].decode("utf8")
+ else:
+ return header[tag]
+
# vim:et:ts=4:sw=4
diff --git a/MgaRepo/simplerpm.py b/MgaRepo/simplerpm.py
index a06e694..3d50176 100644
--- a/MgaRepo/simplerpm.py
+++ b/MgaRepo/simplerpm.py
@@ -7,15 +7,17 @@ class SRPM:
self._getinfo()
def _getinfo(self):
- cmdstr = "rpm -qp --nosignature --qf '%%{name} %%{epoch} %%{release} %%{version}' %s"
- status, output = execcmd(cmdstr % self.filename)
+ args = ["rpm", "-qp", "--qf", "%{name} %{epoch} %{release} %{version}",
+ self.filename]
+ status, output = execcmd(args)
self.name, self.epoch, self.release, self.version = output.split()
if self.epoch == "(none)":
self.epoch = None
def unpack(self, topdir):
- execcmd(("rpm -i --nodeps --define '_sourcedir %s/SOURCES' " +
- "--define '_specdir %s/SPECS' --define '_patchdir %s/SOURCES' %s")
- % (topdir, topdir, topdir, self.filename))
+ args ["rpm", "-i", "--nodeps", "--define", "_sourcedir", topdir+"/SOURCES",
+ "--define", "_specdir", topdir + "/SPECS", "--define", "_patchdir",
+ topdir+"/SOURCES", self.filename]
+ execcmd(args)
# vim:et:ts=4:sw=4
diff --git a/MgaRepo/svn.py b/MgaRepo/svn.py
index 74f47a3..5f8851c 100644
--- a/MgaRepo/svn.py
+++ b/MgaRepo/svn.py
@@ -1,5 +1,5 @@
-from MgaRepo import Error, SilentError, config
-from MgaRepo.util import execcmd, get_auth
+from MgaRepo.util import execcmd
+from MgaRepo.VCS import *
import sys
import os
import re
@@ -7,445 +7,21 @@ import time
__all__ = ["SVN", "SVNLook", "SVNLogEntry"]
-class SVNLogEntry:
+class SVNLogEntry(VCSLogEntry):
def __init__(self, revision, author, date):
- self.revision = revision
- self.author = author
- self.date = date
- self.changed = []
- self.lines = []
+ VCSLogEntry.__init__(self, revision, author, data)
- def __lt__(self, other):
- return (self.date < other.date)
-
- def __eq__(self,other):
- return (self.date == other.date)
+class SVN(VCS):
+ vcs_dirname = ".svn"
+ vcs_name = "svn"
+ def __init__(self, path=None, url=None):
+ VCS.__init__(self, path, url)
+ self.vcs_command = config.get("global", "svn-command", ["svn"])
+ self.env_defaults = {"SVN_SSH": self.vcs_wrapper}
-class SVN:
- def _execsvn(self, *args, **kwargs):
- localcmds = ("add", "revert", "cleanup", "mv")
- if not kwargs.get("show") and args[0] not in localcmds:
- args = list(args)
- args.append("--non-interactive")
- else:
- if args[0] == "mv":
- kwargs["geterr"] = False
- else:
- kwargs["geterr"] = True
- kwargs["cleanerr"] = True
- if kwargs.get("xml"):
- args.append("--xml")
- self._set_env()
- svn_command = config.get("global", "svn-command", "svn")
- cmdstr = svn_command + " " + " ".join(args)
- try:
- if args[0] in ('info', 'checkout','log'):
- kwargs['info'] = True
- else:
- kwargs['info'] = False
- return execcmd(cmdstr, **kwargs)
- except Error as e:
- msg = None
- if e.args:
- if "Permission denied" in e.args:
- msg = ("It seems ssh-agent or ForwardAgent are not setup "
- "or your username is wrong. See "
- "https://wiki.mageia.org/en/Packagers_ssh"
- " for more information.")
- elif "authorization failed" in e.args:
- msg = ("Note that mgarepo does not support any HTTP "
- "authenticated access.")
- if kwargs.get("show") and \
- not config.getbool("global", "verbose", 0):
- # svn has already dumped error messages, we don't need to
- # do it too
- if msg:
- sys.stderr.write("\n")
- sys.stderr.write(msg)
- sys.stderr.write("\n")
- raise SilentError
- elif msg:
- raise Error("%s\n%s" % (e, msg))
- raise
-
- def _set_env(self):
- wrapper = "mgarepo-ssh"
- repsys = config.get("global", "mgarepo-cmd")
- if repsys:
- dir = os.path.dirname(repsys)
- path = os.path.join(dir, wrapper)
- if os.path.exists(path):
- wrapper = path
- defaults = {"SVN_SSH": wrapper}
- os.environ.update(defaults)
- raw = config.get("global", "svn-env")
- if raw:
- for line in raw.split("\n"):
- env = line.strip()
- if not env:
- continue
- try:
- name, value = env.split("=", 1)
- except ValueError:
- sys.stderr.write("invalid svn environment line: %r\n" % env)
- continue
- os.environ[name] = value
-
- def _execsvn_success(self, *args, **kwargs):
- status, output = self._execsvn(*args, **kwargs)
- return status == 0
-
- def _add_log(self, cmd_args, received_kwargs, optional=0):
- if (not optional or
- "log" in received_kwargs or
- "logfile" in received_kwargs):
- ret = received_kwargs.get("log")
- if ret is not None:
- cmd_args.append("-m '%s'" % ret)
- ret = received_kwargs.get("logfile")
- if ret is not None:
- cmd_args.append("-F '%s'" % ret)
-
- def _add_revision(self, cmd_args, received_kwargs, optional=0):
- if not optional or "rev" in received_kwargs:
- ret = received_kwargs.get("rev")
- if isinstance(ret, str):
- if not ret.startswith("{"): # if not a datespec
- try:
- ret = int(ret)
- except ValueError:
- raise Error("invalid revision provided")
- if ret:
- cmd_args.append("-r '%s'" % ret)
-
- def add(self, path, **kwargs):
- cmd = ["add", path + '@' if '@' in path else path]
- return self._execsvn_success(noauth=1, *cmd, **kwargs)
-
- def copy(self, pathfrom, pathto, **kwargs):
- cmd = ["copy", pathfrom + '@' if '@' in pathfrom else pathfrom, pathto + '@' if '@' in pathto else pathto]
- self._add_revision(cmd, kwargs, optional=1)
- self._add_log(cmd, kwargs)
- return self._execsvn_success(*cmd, **kwargs)
-
- def remove(self, path, force=0, **kwargs):
- cmd = ["remove", path + '@' if '@' in path else path]
- self._add_log(cmd, kwargs)
- if force:
- cmd.append("--force")
- return self._execsvn_success(*cmd, **kwargs)
-
- def mkdir(self, path, **kwargs):
- cmd = ["mkdir", path + '@' if '@' in path else path]
- if kwargs.get("parents"):
- cmd.append("--parents")
- self._add_log(cmd, kwargs)
- return self._execsvn_success(*cmd, **kwargs)
-
- def _execsvn_commit(self, *cmd, **kwargs):
- status, output = self._execsvn(*cmd, **kwargs)
- match = re.search("Committed revision (?P<rev>\\d+)\\.$", output)
- if match:
- rawrev = match.group("rev")
- return int(rawrev)
-
- def commit(self, path, **kwargs):
- cmd = ["commit", path + '@' if '@' in path else path]
- if kwargs.get("nonrecursive"):
- cmd.append("-N")
- self._add_log(cmd, kwargs)
- return self._execsvn_commit(*cmd, **kwargs)
-
- def import_(self, path, url, **kwargs):
- cmd = ["import", "'%s'" % path, "'%s'" % url]
- self._add_log(cmd, kwargs)
- return self._execsvn_commit(*cmd, **kwargs)
-
- def export(self, url, targetpath, **kwargs):
- cmd = ["export", "'%s'" % url, targetpath]
- self._add_revision(cmd, kwargs, optional=1)
- return self._execsvn_success(*cmd, **kwargs)
-
- def checkout(self, url, targetpath, **kwargs):
- cmd = ["checkout", "'%s'" % url, targetpath]
- self._add_revision(cmd, kwargs, optional=1)
- return self._execsvn_success(*cmd, **kwargs)
-
- def propget(self, propname, targets, **kwargs):
- cmd = ["propget", propname, targets]
- if kwargs.get("revprop"):
- cmd.append("--revprop")
- self._add_revision(cmd, kwargs)
- status, output = self._execsvn(local=True, *cmd, **kwargs)
- return output
-
- def propset(self, propname, value, targets, **kwargs):
- cmd = ["propset", propname, "'%s'" % value, targets]
- return self._execsvn_success(*cmd, **kwargs)
-
- def propedit(self, propname, target, **kwargs):
- cmd = ["propedit", propname, target]
- if kwargs.get("rev"):
- cmd.append("--revprop")
- self._add_revision(cmd, kwargs)
- return self._execsvn_success(local=True, show=True, *cmd, **kwargs)
-
- def revision(self, path, **kwargs):
- cmd = ["info", path + '@' if '@' in path else path]
- status, output = self._execsvn(local=True, *cmd, **kwargs)
- if status == 0:
- for line in output.splitlines():
- if line.startswith("Last Changed Rev: "):
- return int(line.split()[3])
- return None
-
- def info(self, path, **kwargs):
- cmd = ["info", path + '@' if '@' in path else path]
- status, output = self._execsvn(local=True, noerror=True, *cmd, **kwargs)
- if (("Not a versioned resource" not in output) and ("svn: warning: W155010" not in output)):
- return output.splitlines()
- return None
-
- def info2(self, *args, **kwargs):
- lines = self.info(*args, **kwargs)
- if lines is None:
- return None
- pairs = [[w.strip() for w in line.split(":", 1)] for line in lines]
- info = {}
- for pair in pairs:
- if pair != ['']:
- info[pair[0]]=pair[1]
- return info
-
- def ls(self, path, **kwargs):
- cmd = ["ls", path + '@' if '@' in path else path]
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return output.split()
- return None
-
- def status(self, path, **kwargs):
- cmd = ["status", path + '@' if '@' in path else path]
- if kwargs.get("verbose"):
- cmd.append("-v")
- if kwargs.get("noignore"):
- cmd.append("--no-ignore")
- if kwargs.get("quiet"):
- cmd.append("--quiet")
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return [(x[0], x[8:]) for x in output.splitlines()]
- return None
-
- def cleanup(self, path, **kwargs):
- cmd = ["cleanup", path + '@' if '@' in path else path]
- return self._execsvn_success(*cmd, **kwargs)
-
- def revert(self, path, **kwargs):
- cmd = ["revert", path + '@' if '@' in path else path]
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return [x.split() for x in output.split()]
- return None
-
- def switch(self, url, oldurl=None, path=None, relocate=False, **kwargs):
- cmd = ["switch"]
- if relocate:
- if oldurl is None:
- raise Error("You must supply the old URL when "\
- "relocating working copies")
- cmd.append("--relocate")
- cmd.append(oldurl)
- cmd.append(url)
- if path is not None:
- cmd.append(path)
- return self._execsvn_success(*cmd, **kwargs)
-
- def update(self, path, **kwargs):
- cmd = ["update", path + '@' if '@' in path else path]
- self._add_revision(cmd, kwargs, optional=1)
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return [x.split() for x in output.split()]
- return None
-
- def merge(self, url1, url2=None, rev1=None, rev2=None, path=None,
- **kwargs):
- cmd = ["merge"]
- if rev1 and rev2 and not url2:
- cmd.append("-r")
- cmd.append("%s:%s" % (rev1, rev2))
- cmd.append(url1)
- else:
- if not url2:
- raise ValueError("url2 needed if two revisions are not provided")
- if rev1:
- cmd.append("%s@%s" % (url1, rev1))
- else:
- cmd.append(url1)
- if rev2:
- cmd.append("%s@%s" % (url2, rev2))
- else:
- cmd.append(url2)
- if path:
- cmd.append(path)
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return [x.split() for x in output.split()]
- return None
-
- def diff(self, pathurl1, pathurl2=None, **kwargs):
- cmd = ["diff", pathurl1]
- self._add_revision(cmd, kwargs, optional=1)
- if pathurl2:
- cmd.append(pathurl2)
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return output
- return None
-
- def cat(self, url, **kwargs):
- cmd = ["cat", url]
- self._add_revision(cmd, kwargs, optional=1)
- status, output = self._execsvn(*cmd, **kwargs)
- if status == 0:
- return output
- return None
-
- def log(self, url, start=None, end=0, limit=None, **kwargs):
- cmd = ["log", "-v", url]
- if start is not None or end != 0:
- if start is not None and type(start) is not type(0):
- try:
- start = int(start)
- except (ValueError, TypeError):
- raise Error("invalid log start revision provided")
- if type(end) is not type(0):
- try:
- end = int(end)
- except (ValueError, TypeError):
- raise Error("invalid log end revision provided")
- start = start or "HEAD"
- cmd.append("-r %s:%s" % (start, end))
- if limit is not None:
- try:
- limit = int(limit)
- except (ValueError, TypeError):
- raise Error("invalid limit number provided")
- cmd.append("--limit %d" % limit)
- status, output = self._execsvn(*cmd, **kwargs)
- if status != 0:
- return None
-
- revheader = re.compile("^r(?P<revision>[0-9]+) \| (?P<author>[^\|]+) \| (?P<date>[^\|]+) \| (?P<lines>[0-9]+) (?:line|lines)$")
- changedpat = re.compile(r"^\s+(?P<action>[^\s]+) (?P<path>[^\s]+)(?: \([^\s]+ (?P<from_path>[^:]+)(?:\:(?P<from_rev>[0-9]+))?\))?$")
- logseparator = "-"*72
- linesleft = 0
- entry = None
- log = []
- appendchanged = 0
- changedheader = 0
- for line in output.splitlines():
- line = line.rstrip()
- if changedheader:
- appendchanged = 1
- changedheader = 0
- elif appendchanged:
- if not line:
- appendchanged = 0
- continue
- m = changedpat.match(line)
- if m:
- changed = m.groupdict().copy()
- from_rev = changed.get("from_rev")
- if from_rev is not None:
- try:
- changed["from_rev"] = int(from_rev)
- except (ValueError, TypeError):
- raise Error("invalid revision number in svn log")
- entry.changed.append(changed)
- elif linesleft == 0:
- if line != logseparator:
- m = revheader.match(line)
- if m:
- linesleft = int(m.group("lines"))
- timestr = " ".join(m.group("date").split()[:2])
- timetuple = time.strptime(timestr,
- "%Y-%m-%d %H:%M:%S")
- entry = SVNLogEntry(int(m.group("revision")),
- m.group("author"), timetuple)
- log.append(entry)
- changedheader = 1
- else:
- entry.lines.append(line)
- linesleft -= 1
- log.sort()
- log.reverse()
- return log
-
- def mv(self, path, dest, message=None, **kwargs):
- cmd = ["mv", path, dest, ]
- if message:
- cmd.append("-m '%s'"%message)
- else:
- kwargs['show'] = True
- self._add_log(cmd, kwargs)
- return self._execsvn_success(*cmd, **kwargs)
-
-class SVNLook:
+class SVNLook(VCSLook):
def __init__(self, repospath, txn=None, rev=None):
- self.repospath = repospath
- self.txn = txn
- self.rev = rev
-
- def _execsvnlook(self, cmd, *args, **kwargs):
- execcmd_args = ["svnlook", cmd, self.repospath]
- self._add_txnrev(execcmd_args, kwargs)
- execcmd_args += args
- execcmd_kwargs = {}
- keywords = ["show", "noerror"]
- for key in keywords:
- if key in kwargs:
- execcmd_kwargs[key] = kwargs[key]
- return execcmd(*execcmd_args, **execcmd_kwargs)
-
- def _add_txnrev(self, cmd_args, received_kwargs):
- if "txn" in received_kwargs:
- txn = received_kwargs.get("txn")
- if txn is not None:
- cmd_args += ["-t", txn]
- elif self.txn is not None:
- cmd_args += ["-t", self.txn]
- if "rev" in received_kwargs:
- rev = received_kwargs.get("rev")
- if rev is not None:
- cmd_args += ["-r", rev]
- elif self.rev is not None:
- cmd_args += ["-r", self.rev]
-
- def changed(self, **kwargs):
- status, output = self._execsvnlook("changed", **kwargs)
- if status != 0:
- return None
- changes = []
- for line in output.splitlines():
- line = line.rstrip()
- if not line:
- continue
- entry = [None, None, None]
- changedata, changeprop, path = None, None, None
- if line[0] != "_":
- changedata = line[0]
- if line[1] != " ":
- changeprop = line[1]
- path = line[4:]
- changes.append((changedata, changeprop, path))
- return changes
-
- def author(self, **kwargs):
- status, output = self._execsvnlook("author", **kwargs)
- if status != 0:
- return None
- return output.strip()
+ VCSLook.__init__(self, repospath, txn, rev)
+ self.execcmd = "svnlook"
# vim:et:ts=4:sw=4
diff --git a/MgaRepo/util.py b/MgaRepo/util.py
index 538eae8..64faaa1 100644
--- a/MgaRepo/util.py
+++ b/MgaRepo/util.py
@@ -2,6 +2,7 @@
from MgaRepo import Error, config
+import shlex
import subprocess
import getpass
import sys
@@ -11,73 +12,89 @@ import select
from io import StringIO
import httplib2
-# Our own version of commands' commands_exec(). We have a commands
-# module directory, so we can't import Python's standard module
-
-# Our own version of commands' getstatusoutput(). We have a commands
-# module directory, so we can't import Python's standard module
-def commands_getstatusoutput(cmd):
- """Return (status, output) of executing cmd in a shell."""
- pipe = subprocess.Popen('{ ' + cmd + '; } 2>&1', stdin = subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines = True, shell = True)
- of = pipe.stdout.fileno()
- text = ''
- pipe.stdin.close()
- while True:
- text += os.read(of,8192).decode('utf8')
- status = pipe.poll()
- if status is not None or text == '':
- break
- if text[-1:] == '\n': text = text[:-1]
- return status, text
+class CommandError(Error):
+ def __init__(self, cmdline, status, output):
+ self.cmdline = cmdline
+ self.status = status
+ self.output = output
+
+ def __str__(self):
+ return "command failed: %s\n%s\n" % (self.cmdline, self.output)
def execcmd(*cmd, **kwargs):
- cmdstr = " ".join(cmd)
- verbose = config.getbool("global", "verbose", 0)
- if kwargs.get('info') :
- prefix='LANGUAGE=C LC_TIME=C '
- else:
- prefix='LANG=C LANGUAGE=C LC_ALL=C '
- if verbose:
- print(prefix + cmdstr)
- if kwargs.get("show"):
- if kwargs.get("geterr"):
- err = StringIO()
- pstdin = kwargs.get("stdin") if kwargs.get("stdin") else None
- p = subprocess.Popen(prefix + cmdstr, shell=True,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- stdin=pstdin)
- of = p.stdout.fileno()
- ef = p.stderr.fileno()
- while True:
- r,w,x = select.select((of,ef), (), ())
- odata = None
- if of in r:
- odata = (os.read(of, 8192)).decode('utf8')
- sys.stdout.write(odata)
- edata = None
- if ef in r:
- edata = (os.read(ef, 8192)).decode('utf8')
- err.write(edata)
- sys.stderr.write(edata)
-
- status = p.poll()
- if status is not None and odata == '' and edata == '':
- break
- output = err.getvalue()
+ assert (kwargs.get("collecterr") and kwargs.get("show")) or not kwargs.get("collecterr"), \
+ ("execcmd is implemented to handle collecterr=True only if show=True")
+ # split command args
+ if isinstance(cmd[0], str):
+ if len(cmd) is 1:
+ cmdargs = shlex.split(cmd[0])
else:
- status = os.system(cmdstr)
- output = ""
+ cmdargs = cmd[:]
else:
- status, output = commands_getstatusoutput(prefix + cmdstr)
- if status != 0 and not kwargs.get("noerror"):
- if kwargs.get("cleanerr") and not verbose:
- raise Error(output)
+ cmdargs = cmd[0][:]
+
+ stdout = None
+ stderr = None
+ env = {}
+ env.update(os.environ)
+ if kwargs.get("info") or not kwargs.get("show") or (kwargs.get("show") and kwargs.get("collecterr")):
+ if kwargs.get("info"):
+ env.update({"LANGUAGE": "C", "LC_TIME": "C"})
+ else:
+ env.update({"LANG": "C", "LANGUAGE": "C", "LC_ALL": "C"})
+ stdout = subprocess.PIPE
+ if kwargs.get("collecterr"):
+ stderr = subprocess.PIPE
else:
- raise Error("command failed: %s\n%s\n" % (cmdstr, output))
+ stderr = subprocess.STDOUT
+
+ verbose = config.getbool("global", "verbose", 0)
if verbose:
- print(output)
- sys.stdout.write(output)
- return status, output
+ print("cmd: " + str(cmd).lstrip("(").rstrip(")").replace("', '", "' '"))
+
+ proc = subprocess.Popen(cmdargs, shell=False, stdout=stdout,
+ stderr=stderr, env=env)
+
+ output = ""
+
+ if kwargs.get("show") and kwargs.get("collecterr"):
+ error = StringIO()
+ wl = []
+ outfd = proc.stdout.fileno()
+ errfd = proc.stderr.fileno()
+ rl = [outfd, errfd]
+ xl = wl
+ while proc.poll() is None:
+ mrl, _, _ = select.select(rl, wl, xl, 0.5)
+ for fd in mrl:
+ data = os.read(fd, 8192).decode('utf8')
+ if fd == errfd:
+ error.write(data)
+ sys.stderr.write(data)
+ else:
+ sys.stdout.write(data)
+ output = error.getvalue()
+ else:
+ # Make sure to avoid buffer getting full.
+ # Otherwise if ie. using proc.wait() both python
+ # and the process will just hang
+ while proc.poll() is None:
+ if proc.stdout is not None:
+ output += proc.stdout.read(8192).decode('utf8')
+ # Make sure that we've emptied the buffer entirely
+ if proc.stdout is not None:
+ output += proc.stdout.read().decode('utf8')
+
+ if kwargs.get("strip", True):
+ output = output.rstrip()
+
+ if (not kwargs.get("noerror")) and proc.returncode != 0:
+ if kwargs.get("cleanerr"):
+ msg = output
+ cmdline = subprocess.list2cmdline(cmdargs)
+ raise CommandError(cmdline, proc.returncode, output)
+
+ return proc.returncode, output
def get_output_exec(cmdstr):
output = StringIO()
diff --git a/create-srpm b/create-srpm
index 497ecd6..a2418c2 100755
--- a/create-srpm
+++ b/create-srpm
@@ -10,7 +10,6 @@ import sys
import os
import pwd
import optparse
-import subprocess
import urllib.parse
import urllib.request, urllib.parse, urllib.error
import rpm
@@ -120,8 +119,7 @@ class CmdIface:
upload_command.append(x)
upload_command.append(targetname)
upload_command.extend(uploadsrpms)
- command = subprocess.list2cmdline(upload_command)
- status, output = execcmd(command, noerror=1)
+ status, output = execcmd(upload_command, noerror=1)
for srpm in uploadsrpms:
if os.path.isfile(srpm):
os.unlink(srpm)
diff --git a/mgarepo b/mgarepo
index 7df3daf..2ad21b4 100755
--- a/mgarepo
+++ b/mgarepo
@@ -13,6 +13,7 @@ Tool to access and manage a package repository structure.
Useful commands:
co checkout a package
+ clone clone a package repository
ci commit changes
sync add-remove all file changes from the .spec
submit submit a package for build
diff --git a/mgarepo.conf b/mgarepo.conf
index 932fa0d..bbb1d62 100644
--- a/mgarepo.conf
+++ b/mgarepo.conf
@@ -31,3 +31,7 @@ upload_host = binrepo.mageia.org
host = maintdb.mageia.org
url = http://maintdb.mageia.org/
+[github]
+#login = someuser
+#password = somepassword
+organization = mdkcauldron
diff --git a/setup.py b/setup.py
index 6a5c853..f9195b1 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
#!/usr/bin/python3
-from distutils.core import setup
+from setuptools import setup
import sys
import re
@@ -27,7 +27,8 @@ setup(name="mgarepo",
("share/bash-completion/completions",
["bash-completion/mgarepo"]),
("/etc/", ["mgarepo.conf"]),
- ("share/man/man8/", ["mgarepo.8"])]
+ ("share/man/man8/", ["mgarepo.8"])],
+ install_requires=['PyGithub>=1.26.0']
)
# vim:ts=4:sw=4:et
generated by cgit v1.2.1 (git 2.21.0) at 2024-05-07 16:05:35 +0000