diff options
Diffstat (limited to 'MgaRepo')
38 files changed, 657 insertions, 1403 deletions
diff --git a/MgaRepo/GitHub.py b/MgaRepo/GitHub.py deleted file mode 100644 index 74da43e..0000000 --- a/MgaRepo/GitHub.py +++ /dev/null @@ -1,90 +0,0 @@ -from MgaRepo import Error, config -from MgaRepo.rpmutil import get_pkg_tag, clone -from MgaRepo.util import execcmd -from MgaRepo import layout -from MgaRepo.git import GIT -from MgaRepo.svn import SVN -from MgaRepo.vcsutil import detectVCS -from rpm import RPMTAG_SUMMARY, RPMTAG_URL -import github -import os - -class GitHub(object): - def __init__(self, username = config.get("github", "login"), password = config.get("github", "password")): - self._github = github.Github(login_or_token=username, password=password) - self._organization = self._github.get_organization(config.get("github", "organization", "mdkcauldron")) - self._repos = self._organization.get_repos() - - def repository_exists(self, name): - for repo in self._repos: - if repo.name == name: - return repo - return None - - def create_repository(self, pkgname, **kwargs): - repository = self._organization.create_repo(pkgname, **kwargs) - return repository - - def delete_repository(self, pkgname, **kwargs): - repository = self.repository_exists(pkgname) - if repository: - print("deleting repository %s" % repository.full_name) - repository.delete() - return True - raise Error("repository %s doesn't exist!" % (self._organization.login+"/"+pkgname)) - - def import_package(self, target): - if not os.path.exists(target): - target = layout.checkout_url(layout.package_url(target)) - vcs = detectVCS(target) - top_dir = vcs.get_topdir() - pkgname = layout.package_name(layout.remove_current(vcs.url)) - - repository = self.repository_exists(pkgname) - if not repository or not repository.get_stats_commit_activity(): - if not repository: - if os.path.exists(vcs.path): - summary = get_pkg_tag(RPMTAG_SUMMARY, path=top_dir) - url = get_pkg_tag(RPMTAG_URL, path=top_dir) - repository = self.create_repository(pkgname, description=summary, homepage=url) - print("GitHub repository created at " + repository.html_url) - else: - print("Empty GitHub repository already created at %s, using" % repository.html_url) - - if isinstance(vcs, GIT): - status, output = vcs.remote("add", repository.full_name, repository.ssh_url, noerror=True) - if status: - if status == 128 and ("fatal: remote %s already exists." % repository.full_name) \ - in output: - pass - else: - raise Error(output) - - status, output = vcs.push("--mirror", repository.full_name, show=True) - if status == 0: - print("Success!") - return True - elif isinstance(vcs, SVN): - clone(vcs.url, bindownload=False) - return self.import_package(pkgname) - - else: - raise Error("GitHub repository already exists at " + repository.html_url) - raise Error("GitHub import failed...") - - def clone_repository(self, pkgname, target=None): - if not target: - target = pkgname - repository = self.repository_exists(pkgname) - if repository: - svnurl = layout.checkout_url(layout.package_url(pkgname)) - if repository.permissions: - giturl = repository.ssh_url - else: - giturl = repository.git_url - execcmd(("git", "clone", "--mirror", giturl, os.path.join(target, ".git")), show=True) - git_svn = GIT(path=target, url=svnurl) - git_svn.init(svnurl, pkgname, branch="master", fullnames=True) - - return True - raise Error("Repository %s doesn't exist!" % (self._organization.login+"/"+pkgname)) diff --git a/MgaRepo/VCS.py b/MgaRepo/VCS.py deleted file mode 100644 index 4edbd7f..0000000 --- a/MgaRepo/VCS.py +++ /dev/null @@ -1,475 +0,0 @@ -from MgaRepo import Error, SilentError, config -from MgaRepo.util import execcmd, get_auth -from MgaRepo import layout -from xml.etree import ElementTree -import sys -import os -import re -import time - -class VCSLogEntry(object): - def __init__(self, revision, author, date, lines=[], changed=[]): - self.revision = revision - self.author = author - self.date = date - self.changed = changed - self.lines = lines - - def __lt__(self, other): - return (self.date < other.date) - - def __eq__(self,other): - return (self.date == other.date) - -class VCS(object): - vcs_dirname = None - vcs_name = None - def __init__(self, path, url): - self.vcs_command = None - self.vcs_wrapper = "mga-ssh" - self.vcs_supports = {'clone' : False} - self.vcs_type = None - self.env_defaults = None - if not path and not url: - self._path = os.path.curdir - elif not path and url: - self._path = layout.package_name(layout.remove_current(url)) - else: - self._path = path - # FIXME - self._url = None - self.__url = url - - def _execVcs(self, *args, **kwargs): - localcmds = ("add", "revert", "cleanup", "mv") - cmd = self.vcs_command + list(args) - kwargs["collecterr"] = kwargs.get("collecterr", False) - if kwargs.get("show"): - if not kwargs.get("local"): - kwargs["collecterr"] = True - else: - if self.vcs_command is "svn" and args[0] not in localcmds: - cmd.append("--non-interactive") - else: - if args[0] == "mv": - kwargs["collecterr"] = False - kwargs["cleanerr"] = kwargs.get("cleanerr", True) - if kwargs.get("xml"): - cmd.append("--xml") - try: - if args[0] in ('info', 'checkout','log'): - kwargs['info'] = True - else: - kwargs['info'] = False - return execcmd(*cmd, **kwargs) - except Error as e: - msg = None - if e.args: - if "Permission denied" in e.args: - msg = ("It seems ssh-agent or ForwardAgent are not setup " - "or your username is wrong. See " - "https://wiki.mageia.org/en/Packagers_ssh" - " for more information.") - elif "authorization failed" in e.args: - msg = ("Note that mgarepo does not support any HTTP " - "authenticated access.") - if kwargs.get("show") and \ - not config.getbool("global", "verbose", 0): - # svn has already dumped error messages, we don't need to - # do it too - if msg: - sys.stderr.write("\n") - sys.stderr.write(msg) - sys.stderr.write("\n") - raise SilentError - elif msg: - raise Error("%s\n%s" % (e, msg)) - raise - - def _set_env(self): - wrapper = "mgarepo-ssh" - repsys = config.get("global", "mgarepo-cmd") - if repsys: - dir = os.path.dirname(repsys) - path = os.path.join(dir, wrapper) - if os.path.exists(path): - wrapper = path - defaults = {"SVN_SSH": wrapper} - os.environ.update(defaults) - raw = config.get("global", "svn-env") - if raw: - for line in raw.split("\n"): - env = line.strip() - if not env: - continue - try: - name, value = env.split("=", 1) - except ValueError: - sys.stderr.write("invalid svn environment line: %r\n" % env) - continue - os.environ[name] = value - - def _execVcs_success(self, *args, **kwargs): - status, output = self._execVcs(*args, **kwargs) - return status == 0 - - def _add_log(self, cmd_args, received_kwargs, optional=0): - if (not optional or - "log" in received_kwargs or - "logfile" in received_kwargs): - ret = received_kwargs.get("log") - if ret is not None: - cmd_args.extend(("-m", ret)) - ret = received_kwargs.get("logfile") - if ret is not None: - cmd_args.extend(("-F", ret)) - - def _add_revision(self, cmd_args, received_kwargs, optional=0): - if not optional or "rev" in received_kwargs: - ret = received_kwargs.get("rev") - if isinstance(ret, str): - if not ret.startswith("{"): # if not a datespec - try: - ret = int(ret) - except ValueError: - raise Error("invalid revision provided") - if ret: - cmd_args.extend(("-r", str(ret))) - - def add(self, path, **kwargs): - cmd = ["add", path + '@' if '@' in path else path] - return self._execVcs_success(noauth=1, *cmd, **kwargs) - - def copy(self, pathfrom, pathto, **kwargs): - cmd = ["copy", pathfrom + '@' if '@' in pathfrom else pathfrom, pathto + '@' if '@' in pathto else pathto] - self._add_revision(cmd, kwargs, optional=1) - self._add_log(cmd, kwargs) - return self._execVcs_success(*cmd, **kwargs) - - def remove(self, path, force=0, **kwargs): - cmd = ["remove", path + '@' if '@' in path else path] - self._add_log(cmd, kwargs) - if force: - cmd.append("--force") - return self._execVcs_success(*cmd, **kwargs) - - def mkdir(self, path, **kwargs): - cmd = ["mkdir", path + '@' if '@' in path else path] - if kwargs.get("parents"): - cmd.append("--parents") - self._add_log(cmd, kwargs) - return self._execVcs_success(*cmd, **kwargs) - - def _execVcs_commit(self, *cmd, **kwargs): - status, output = self._execVcs(*cmd, **kwargs) - match = re.search("Committed revision (?P<rev>\\d+)\\.$", output) - if match: - rawrev = match.group("rev") - return int(rawrev) - - def commit(self, path, **kwargs): - cmd = ["commit", path + '@' if '@' in path else path] - if kwargs.get("nonrecursive"): - cmd.append("-N") - self._add_log(cmd, kwargs) - return self._execVcs_commit(*cmd, **kwargs) - - def import_(self, path, url, **kwargs): - cmd = ["import", path, url] - self._add_log(cmd, kwargs) - return self._execVcs_commit(*cmd, **kwargs) - - def export(self, url, targetpath, **kwargs): - cmd = ["export", url, targetpath] - self._add_revision(cmd, kwargs, optional=1) - return self._execVcs_success(*cmd, **kwargs) - - def checkout(self, url, targetpath, **kwargs): - cmd = ["checkout", url, targetpath] - self._add_revision(cmd, kwargs, optional=1) - return self._execVcs_success(*cmd, **kwargs) - - def clone(self, url, targetpath, **kwargs): - if self.vcs_supports['clone']: - cmd = ["clone", url, targetpath] - return self._execVcs_success(*cmd, **kwargs) - else: - raise Error("%s doesn't support 'clone'" % self.vcs_name) - - def propget(self, propname, targets, **kwargs): - cmd = ["propget", propname, targets] - if kwargs.get("revprop"): - cmd.append("--revprop") - self._add_revision(cmd, kwargs) - status, output = self._execVcs(local=True, *cmd, **kwargs) - return output - - def propset(self, propname, value, targets, **kwargs): - cmd = ["propset", propname, value, targets] - return self._execVcs_success(*cmd, **kwargs) - - def propedit(self, propname, target, **kwargs): - cmd = ["propedit", propname, target] - if kwargs.get("rev"): - cmd.append("--revprop") - self._add_revision(cmd, kwargs) - return self._execVcs_success(local=True, show=True, *cmd, **kwargs) - - def revision(self, path, **kwargs): - cmd = ["info", path + '@' if '@' in path else path] - status, output = self._execVcs(local=True, *cmd, **kwargs) - if status == 0: - for line in output.splitlines(): - if line.startswith("Last Changed Rev: "): - return int(line.split()[3]) - return None - - def info(self, path, **kwargs): - cmd = ["info", path + '@' if '@' in path else path] - status, output = self._execVcs(local=True, noerror=True, *cmd, **kwargs) - if (("Not a versioned resource" not in output) and ("svn: warning: W155010" not in output)): - return output.splitlines() - return None - - def info2(self, *args, **kwargs): - lines = self.info(*args, **kwargs) - if lines is None: - return None - pairs = [[w.strip() for w in line.split(":", 1)] for line in lines] - info = {} - for pair in pairs: - if pair != ['']: - info[pair[0]]=pair[1] - return info - - def ls(self, path, **kwargs): - cmd = ["ls", path + '@' if '@' in path else path] - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return output.split() - return None - - def status(self, path, **kwargs): - cmd = ["status", path + '@' if '@' in path else path] - if kwargs.get("verbose"): - cmd.append("-v") - if kwargs.get("noignore"): - cmd.append("--no-ignore") - if kwargs.get("quiet"): - cmd.append("--quiet") - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return [(x[0], x[8:]) for x in output.splitlines()] - return None - - def cleanup(self, path, **kwargs): - cmd = ["cleanup", path + '@' if '@' in path else path] - return self._execVcs_success(*cmd, **kwargs) - - def revert(self, path, **kwargs): - cmd = ["revert", path + '@' if '@' in path else path] - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return [x.split() for x in output.split()] - return None - - def switch(self, url, oldurl=None, path=None, relocate=False, **kwargs): - cmd = ["switch"] - if relocate: - if oldurl is None: - raise Error("You must supply the old URL when "\ - "relocating working copies") - cmd.append("--relocate") - cmd.append(oldurl) - cmd.append(url) - if path is not None: - cmd.append(path) - return self._execVcs_success(*cmd, **kwargs) - - def update(self, path, **kwargs): - cmd = ["update", path + '@' if '@' in path else path] - self._add_revision(cmd, kwargs, optional=1) - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return [x.split() for x in output.split()] - return None - - def merge(self, url1, url2=None, rev1=None, rev2=None, path=None, - **kwargs): - cmd = ["merge"] - if rev1 and rev2 and not url2: - cmd.append("-r") - cmd.append("%s:%s" % (rev1, rev2)) - cmd.append(url1) - else: - if not url2: - raise ValueError("url2 needed if two revisions are not provided") - if rev1: - cmd.append("%s@%s" % (url1, rev1)) - else: - cmd.append(url1) - if rev2: - cmd.append("%s@%s" % (url2, rev2)) - else: - cmd.append(url2) - if path: - cmd.append(path) - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return [x.split() for x in output.split()] - return None - - def diff(self, pathurl1, pathurl2=None, **kwargs): - cmd = ["diff", pathurl1] - self._add_revision(cmd, kwargs, optional=1) - if pathurl2: - cmd.append(pathurl2) - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return output - return None - - def cat(self, url, **kwargs): - cmd = ["cat", url] - self._add_revision(cmd, kwargs, optional=1) - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return output - return None - - def log(self, url, start=None, end=0, limit=None, **kwargs): - cmd = ["log", "-v", url] - if start is not None or end != 0: - if start is not None and type(start) is not type(0): - try: - start = int(start) - except (ValueError, TypeError): - raise Error("invalid log start revision provided") - if type(end) is not type(0): - try: - end = int(end) - except (ValueError, TypeError): - raise Error("invalid log end revision provided") - start = start or "HEAD" - cmd.extend(("-r", "%s:%s" % (start, end))) - if limit is not None: - try: - limit = int(limit) - except (ValueError, TypeError): - raise Error("invalid limit number provided") - cmd.extend(("--limit", str(limit))) - - status, output = self._execVcs(*cmd, xml=True, **kwargs) - if status != 0: - return None - - xmllog = ElementTree.fromstring(output) - log = [] - logentries = xmllog.getiterator("logentry") - for entry in logentries: - changed = [] - lines = [] - for pathelem in entry.getiterator("paths"): - path = pathelem.find("path") - from_rev = path.get("copyfrom-rev") - if from_rev: - from_rev = int(from_rev) - changed.append({"from_rev" : from_rev, "from_path" : path.get("copyfrom-path"), "action" : path.get("action"), "path" : path.text}) - date = entry.findtext("date").split("T") - timestr = "%s %s" % (date[0], date[1].split(".")[0]) - timetuple = time.strptime(timestr, "%Y-%m-%d %H:%M:%S") - lines.extend(entry.findtext("msg").rstrip().split("\n")) - logentry = VCSLogEntry(int(entry.attrib["revision"]), - entry.findtext("author"), timetuple, [line.rstrip() for line in lines], changed) - log.append(logentry) - log.sort() - log.reverse() - return log - - def mv(self, path, dest, message=None, **kwargs): - cmd = ["mv", path, dest, ] - if message: - cmd.extend(("-m", str(message))) - else: - kwargs['show'] = True - self._add_log(cmd, kwargs) - return self._execVcs_success(*cmd, **kwargs) - - def get_topdir(self): - vcsdir = os.path.join(self._path, self.vcs_dirname) - if os.path.exists(vcsdir) and os.path.isdir(vcsdir): - return self._path - else: - return None - - def drop_ssh_if_no_auth(self, url): - return url - - @property - def path(self): - return self._path - - @property - def url(self): - if not self._url: - self._url = self.drop_ssh_if_no_auth(self.__url or self.info2(self._path)["URL"]) - return self._url - -class VCSLook(object): - def __init__(self, repospath, txn=None, rev=None): - self.repospath = repospath - self.txn = txn - self.rev = rev - self.execcmd = None - - def _execVcslook(self, cmd, *args, **kwargs): - execcmd_args = ["svnlook", cmd, self.repospath] - self._add_txnrev(execcmd_args, kwargs) - execcmd_args += args - execcmd_kwargs = {} - keywords = ["show", "noerror"] - for key in keywords: - if key in kwargs: - execcmd_kwargs[key] = kwargs[key] - return execcmd(*execcmd_args, **execcmd_kwargs) - - def _add_txnrev(self, cmd_args, received_kwargs): - if "txn" in received_kwargs: - txn = received_kwargs.get("txn") - if txn is not None: - cmd_args.extend(("-t", txn)) - elif self.txn is not None: - cmd_args.extend(("-t", self.txn)) - if "rev" in received_kwargs: - rev = received_kwargs.get("rev") - if rev is not None: - cmd_args.exten(("-r", str(rev))) - elif self.rev is not None: - cmd_args.extend(("-r", str(self.rev))) - - def changed(self, **kwargs): - status, output = self._execVcslook("changed", **kwargs) - if status != 0: - return None - changes = [] - for line in output.splitlines(): - line = line.rstrip() - if not line: - continue - entry = [None, None, None] - changedata, changeprop, path = None, None, None - if line[0] != "_": - changedata = line[0] - if line[1] != " ": - changeprop = line[1] - path = line[4:] - changes.append((changedata, changeprop, path)) - return changes - - def author(self, **kwargs): - status, output = self._execVcslook("author", **kwargs) - if status != 0: - return None - return output.strip() - -# vim:et:ts=4:sw=4 diff --git a/MgaRepo/__init__.py b/MgaRepo/__init__.py index 5d26da2..3c39f55 100644 --- a/MgaRepo/__init__.py +++ b/MgaRepo/__init__.py @@ -1,3 +1,4 @@ +#!/usr/bin/python3 import re import os import tempfile diff --git a/MgaRepo/binrepo.py b/MgaRepo/binrepo.py index 0df8088..89679b0 100644 --- a/MgaRepo/binrepo.py +++ b/MgaRepo/binrepo.py @@ -102,12 +102,12 @@ def upload_binary(topdir, filename): return host = config.get("binrepo", "upload_host") upload_bin_helper = get_helper("upload-bin") - command = ["ssh", host, upload_bin_helper, filename] + command = "ssh %s %s %s" % (host, upload_bin_helper, filename) try: filein = open(filepath, 'r') except Error as e: raise Error("Could not open file %s\n" % filepath) - status, output = execcmd(*command, show=True, collecterr=True, stdin=filein) + status, output = execcmd(command, show=True, geterr=True, stdin=filein) def import_binaries(topdir, pkgname): """Import all binaries from a given package checkout diff --git a/MgaRepo/cgi/soapserver.py b/MgaRepo/cgi/soapserver.py index 8702cca..2fdbe0b 100644 --- a/MgaRepo/cgi/soapserver.py +++ b/MgaRepo/cgi/soapserver.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, config from MgaRepo.rpmutil import get_srpm from MgaRepo.cgiutil import CgiError, get_targets diff --git a/MgaRepo/cgi/submit.py b/MgaRepo/cgi/submit.py index 85cb5a8..e06ae07 100644 --- a/MgaRepo/cgi/submit.py +++ b/MgaRepo/cgi/submit.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, config from MgaRepo.rpmutil import get_srpm from MgaRepo.cgiutil import CgiError, get_targets diff --git a/MgaRepo/cgi/xmlrpcserver.py b/MgaRepo/cgi/xmlrpcserver.py index b042a9c..a1b2b73 100644 --- a/MgaRepo/cgi/xmlrpcserver.py +++ b/MgaRepo/cgi/xmlrpcserver.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, config from MgaRepo.rpmutil import get_srpm from MgaRepo.cgiutil import CgiError, get_targets diff --git a/MgaRepo/cgiutil.py b/MgaRepo/cgiutil.py index 072604e..7dcc71b 100644 --- a/MgaRepo/cgiutil.py +++ b/MgaRepo/cgiutil.py @@ -1,3 +1,4 @@ +#!/usr/bin/python3 from MgaRepo import Error, config from MgaRepo.svn import SVN from MgaRepo.ConfigParser import NoSectionError diff --git a/MgaRepo/command.py b/MgaRepo/command.py index ae297d6..72a6894 100644 --- a/MgaRepo/command.py +++ b/MgaRepo/command.py @@ -1,3 +1,4 @@ +#!/usr/bin/python3 from MgaRepo import SilentError, Error, config import sys, os import urllib.parse diff --git a/MgaRepo/commands/authoremail.py b/MgaRepo/commands/authoremail.py index 27ffaf5..fcb8a86 100644 --- a/MgaRepo/commands/authoremail.py +++ b/MgaRepo/commands/authoremail.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, config from MgaRepo.command import * import sys diff --git a/MgaRepo/commands/buildrpm.py b/MgaRepo/commands/buildrpm.py deleted file mode 100644 index a9230e4..0000000 --- a/MgaRepo/commands/buildrpm.py +++ /dev/null @@ -1,43 +0,0 @@ -from MgaRepo.command import do_command -from MgaRepo.rpmutil import build_rpm -from optparse import * - -HELP = """\ -Usage: mgarepo buildrpm [OPTIONS] - -Builds the binary RPM(s) (.rpm) file(s) of a given package. - -Options: - -bX Build stage option, where X is stage, default is -bb - -I Don't automatically try install missing build dependencies - -L Disable rpmlint check of packages built - -P USER Define the RPM packager information to USER - -d Use DNF - -q Quiet build output - -s Jump to specific build stage (--short-circuit) - -l Use subversion log to build rpm %changelog - -F Do not use full name & email for packagers in %changelog - -- Options and arguments following will be passed to rpmbuild - -""" - -def parse_options(): - parser = OptionParser(HELP) - parser.add_option("-b", dest="build_cmd", default="a") - parser.add_option("-I", dest="installdeps", action="store_false", default=True) - parser.add_option("-L", dest="rpmlint", action="store_false", default=True) - parser.add_option("-P", dest="packager", default="") - parser.add_option("-d", "--dnf", dest="use_dnf", action="store_true", default=False) - parser.add_option("-q", "--quiet", dest="verbose", action="store_false", default=True) - parser.add_option("-s", "--short-circuit", dest="short_circuit", action="store_true", default=False) - parser.add_option("-l", dest="svnlog", action="store_true", default=False) - parser.add_option("-F", dest="fullnames", default=True, - action="store_false") - opts, args = parser.parse_args() - opts.rpmargs = parser.rargs - return opts - -def main(): - do_command(parse_options, build_rpm) - -# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/changed.py b/MgaRepo/commands/changed.py index 493e9af..4ac16bc 100644 --- a/MgaRepo/commands/changed.py +++ b/MgaRepo/commands/changed.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, disable_mirror from MgaRepo.command import * from MgaRepo.layout import package_url diff --git a/MgaRepo/commands/ci.py b/MgaRepo/commands/ci.py index 24e7923..e64ac4e 100644 --- a/MgaRepo/commands/ci.py +++ b/MgaRepo/commands/ci.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo.command import * from MgaRepo.rpmutil import commit diff --git a/MgaRepo/commands/clone.py b/MgaRepo/commands/clone.py deleted file mode 100644 index 88279db..0000000 --- a/MgaRepo/commands/clone.py +++ /dev/null @@ -1,53 +0,0 @@ -from MgaRepo import Error -from MgaRepo.command import * -from MgaRepo.rpmutil import clone -import getopt -import sys - -HELP = """\ -Usage: mgarepo co [OPTIONS] URL [LOCALPATH] - -Clone the package source from the Mageia repository. - -If the 'mirror' option is enabled, the package is obtained from the mirror -repository. - -You can specify the distro branch to checkout from by using distro/pkgname. - -Options: - -d The distribution branch to checkout from - -b The package branch - -M Do not use the mirror (use the main repository) - -h Show this message - -F Do not convert svn usernames to full name & email - -Examples: - mgarepo co pkgname - mgarepo co -d 1 pkgname - mgarepo co 1/pkgame - mgarepo co http://repos/svn/cnc/snapshot/foo - mgarepo co http://repos/svn/cnc/snapshot/foo foo-pkg -""" - -def parse_options(): - parser = OptionParser(help=HELP) - parser.add_option("--distribution", "-d", dest="distro", default=None) - parser.add_option("--branch", "-b", dest="branch", default=None) - parser.add_option("-F", dest="fullnames", default=True, - action="store_false") - opts, args = parser.parse_args() - if len(args) not in (1, 2): - raise Error("invalid arguments") - # here we don't use package_url in order to notify the user we are - # using the mirror - opts.pkgdirurl = args[0] - if len(args) == 2: - opts.path = args[1] - else: - opts.path = None - return opts - -def main(): - do_command(parse_options, clone) - -# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/co.py b/MgaRepo/commands/co.py index b257f21..a6911cc 100644 --- a/MgaRepo/commands/co.py +++ b/MgaRepo/commands/co.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, disable_mirror from MgaRepo.command import * from MgaRepo.rpmutil import checkout diff --git a/MgaRepo/commands/create.py b/MgaRepo/commands/create.py index a947fed..e9a89e8 100644 --- a/MgaRepo/commands/create.py +++ b/MgaRepo/commands/create.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error from MgaRepo.command import * from MgaRepo.layout import package_url diff --git a/MgaRepo/commands/editlog.py b/MgaRepo/commands/editlog.py index 9e82a38..163651d 100644 --- a/MgaRepo/commands/editlog.py +++ b/MgaRepo/commands/editlog.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error from MgaRepo.command import * from MgaRepo.layout import package_url diff --git a/MgaRepo/commands/getspec.py b/MgaRepo/commands/getspec.py index 884e919..e1ba4c4 100644 --- a/MgaRepo/commands/getspec.py +++ b/MgaRepo/commands/getspec.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, disable_mirror from MgaRepo.command import * from MgaRepo.layout import package_url diff --git a/MgaRepo/commands/getsrpm.py b/MgaRepo/commands/getsrpm.py index caadfeb..82bd626 100644 --- a/MgaRepo/commands/getsrpm.py +++ b/MgaRepo/commands/getsrpm.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # # This program will extract given version/revision of the named package # from the Conectiva Linux repository system. @@ -29,7 +30,6 @@ Options: -n Rename the package to include the revision number -l Use subversion log to build rpm %changelog -T FILE Template to be used to generate the %changelog - -F Do not use full name & email for packagers in %changelog -M Do not use the mirror (use the main repository) -h Show this message --strict Check if the given revision contains changes in REPPKGURL @@ -76,8 +76,6 @@ def parse_options(): parser.add_option("-n", dest="revname", action="store_true") parser.add_option("-l", dest="svnlog", action="store_true") parser.add_option("-T", dest="template", type="string", default=None) - parser.add_option("-F", dest="fullnames", default=True, - action="store_false") parser.add_option("-M", "--no-mirror", action="callback", callback=disable_mirror) parser.add_option("--strict", dest="strict", default=False, diff --git a/MgaRepo/commands/github.py b/MgaRepo/commands/github.py deleted file mode 100644 index 7d787e4..0000000 --- a/MgaRepo/commands/github.py +++ /dev/null @@ -1,56 +0,0 @@ -from MgaRepo import Error -from MgaRepo.command import * -from MgaRepo.GitHub import GitHub -import getopt -import sys - -HELP = """\ -Usage: mgarepo github [OPTIONS] URL - -Import a git-svn cloned repository to github - -Options: - -h Show this message - -Examples: - mgarepo github import existingpkg - mgarepo github import svn://svn.mageia.org/svn/packages/cauldron/existingpkg -""" -def github_clone(pkg, **kwargs): - github = GitHub() - github.clone_repository(pkg) - -def github_import(target=".", **kwargs): - github = GitHub() - github.import_package(target) - -def github_delete(pkg, **kwargs): - github = GitHub() - github.delete_repository(pkg) - -def parse_options(): - parser = OptionParser(help=HELP) - opts, args = parser.parse_args() - if len(args) < 1: - raise Error("invalid arguments") - opts.func = globals().get("github_"+args[0], None) - if args[0] == "import": - if len(args) > 1: - opts.target = args[1] - elif args[0] == "delete" or args[0] == "clone": - opts.pkg = args[1] - else: - raise Error("invalid arguments: %s" % str(args)) - return opts - -def dispatch_cmd(*args, **kwargs): - func = kwargs.pop("func", None) - if func: - func(**kwargs) - else: - raise Error("invalid command: %s %s" % (sys.argv[0], sys.argv[1])) - -def main(): - do_command(parse_options, dispatch_cmd) - -# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/log.py b/MgaRepo/commands/log.py index 6d742de..2181125 100644 --- a/MgaRepo/commands/log.py +++ b/MgaRepo/commands/log.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import config, mirror, disable_mirror from MgaRepo.command import * from MgaRepo.layout import package_url, checkout_url @@ -5,8 +6,6 @@ from MgaRepo.rpmutil import sync from MgaRepo.util import execcmd import sys import os -import subprocess -import shlex HELP = """\ Usage: mgarepo log [OPTIONS] [PACKAGE] @@ -58,13 +57,9 @@ def svn_log(pkgdirurl, verbose=False, limit=None, revision=None, releases=None): args.append("-r") args.append(revision) if os.isatty(sys.stdin.fileno()): - pager = shlex.split(os.environ.get("PAGER", "less")) - p = subprocess.Popen(args, stdout=subprocess.PIPE) - p2 = subprocess.Popen(pager, stdin=p.stdout) - p2.wait() - p.wait() - else: - execcmd(args, show=True) + args.append("| less") + rawcmd = " ".join(args) + execcmd(rawcmd, show=True) def main(): do_command(parse_options, svn_log) diff --git a/MgaRepo/commands/maintdb.py b/MgaRepo/commands/maintdb.py index 0bd4d56..9a97be8 100644 --- a/MgaRepo/commands/maintdb.py +++ b/MgaRepo/commands/maintdb.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, config from MgaRepo.command import * from MgaRepo.util import execcmd, get_helper @@ -32,7 +33,8 @@ def parse_options(): def maintdb(maintdb_args): host = config.get("maintdb", "host", "maintdb.mageia.org") maintdb_helper = get_helper("maintdb") - command = ["ssh", host, maintdb_helper] + maintdb_args + cmd_args = ' '.join(maintdb_args) + command = "ssh %s %s %s" % (host, maintdb_helper, cmd_args) execcmd(command, show=True) sys.exit(0) diff --git a/MgaRepo/commands/markrelease.py b/MgaRepo/commands/markrelease.py index 6fb5bbf..534d87f 100644 --- a/MgaRepo/commands/markrelease.py +++ b/MgaRepo/commands/markrelease.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # # This program will append a release to the Conectiva Linux package # repository system. It's meant to be a startup system to include diff --git a/MgaRepo/commands/obsolete.py b/MgaRepo/commands/obsolete.py index 9477721..04d8240 100644 --- a/MgaRepo/commands/obsolete.py +++ b/MgaRepo/commands/obsolete.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # # This program will try to patch a spec file from a given package url. # diff --git a/MgaRepo/commands/patchspec.py b/MgaRepo/commands/patchspec.py index 2fd8da4..dfe54dc 100644 --- a/MgaRepo/commands/patchspec.py +++ b/MgaRepo/commands/patchspec.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # # This program will try to patch a spec file from a given package url. # diff --git a/MgaRepo/commands/putsrpm.py b/MgaRepo/commands/putsrpm.py index aa4043c..242423b 100644 --- a/MgaRepo/commands/putsrpm.py +++ b/MgaRepo/commands/putsrpm.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error from MgaRepo.command import * from MgaRepo.layout import package_url diff --git a/MgaRepo/commands/rpmlog.py b/MgaRepo/commands/rpmlog.py index 693fe8b..88dfc4b 100644 --- a/MgaRepo/commands/rpmlog.py +++ b/MgaRepo/commands/rpmlog.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # # This program will convert the output of "svn log" to be suitable # for usage in an rpm %changelog session. @@ -24,7 +25,6 @@ Options: -p Append changelog found in .spec file -s Sort changelog entries, even from the old log -M Do not use the mirror (use the main repository) - -F Do not use full name & email for packagers where available -h Show this message Examples: @@ -45,15 +45,13 @@ def parse_options(): action="store_true") parser.add_option("-M", "--no-mirror", action="callback", callback=disable_mirror) - parser.add_option("-F", dest="fullnames", default=True, - action="store_false") opts, args = parser.parse_args() if len(args) != 1: raise Error("invalid arguments") opts.pkgdirurl = layout.package_url(args[0]) return opts -def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort, fullnames): +def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort): another = None if usespec: svn = SVN() @@ -61,10 +59,7 @@ def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort, fullnames rawspec = svn.cat(specurl, rev=revision) spec, another = split_spec_changelog(StringIO(rawspec)) newlog = get_changelog(pkgdirurl, another=another, rev=revision, - size=size, sort=sort, template=template, oldlog=oldlog, fullnames=fullnames) - # make sure stdout support unicode, otherwise it'll croak when encountered - if not "UTF-8" in sys.stdout.encoding: - sys.stdout = open(sys.stdout.fileno(), mode="w", encoding="UTF-8") + size=size, sort=sort, template=template, oldlog=oldlog) sys.stdout.writelines(newlog) def main(): diff --git a/MgaRepo/commands/submit.py b/MgaRepo/commands/submit.py index 665b98a..9f05dca 100644 --- a/MgaRepo/commands/submit.py +++ b/MgaRepo/commands/submit.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo import Error, config, layout, mirror from MgaRepo.svn import SVN from MgaRepo.command import * @@ -159,10 +160,9 @@ def list_targets(option, opt, val, parser): raise Error("no submit host defined in mgarepo.conf") createsrpm = get_helper("create-srpm") #TODO make it configurable - args = ["ssh", host, createsrpm, "--list"] - execcmd(args, show=true) - sys.exit(0) # it is invoked via optparse callback, thus we need to - # force ending the script + command = "ssh %s %s --list" % (host, createsrpm) + execcmd(command, show=True) + sys.exit(0) def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): if submithost is None: @@ -197,7 +197,8 @@ def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): else: cmdsargs.extend((baseargs + [url]) for url in urls) for cmdargs in cmdsargs: - status, output = execcmd(cmdargs) + command = subprocess.list2cmdline(cmdargs) + status, output = execcmd(command) if status == 0: print("Package submitted!") else: diff --git a/MgaRepo/commands/switch.py b/MgaRepo/commands/switch.py index 8a33969..ccca76e 100644 --- a/MgaRepo/commands/switch.py +++ b/MgaRepo/commands/switch.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo.command import * from MgaRepo.rpmutil import switch diff --git a/MgaRepo/commands/sync.py b/MgaRepo/commands/sync.py index a56a36e..54f5635 100644 --- a/MgaRepo/commands/sync.py +++ b/MgaRepo/commands/sync.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from MgaRepo.command import * from MgaRepo.rpmutil import sync diff --git a/MgaRepo/git.py b/MgaRepo/git.py deleted file mode 100644 index 670bfa7..0000000 --- a/MgaRepo/git.py +++ /dev/null @@ -1,194 +0,0 @@ -from MgaRepo import Error, config -from MgaRepo.util import execcmd -from MgaRepo.VCS import * -from MgaRepo.svn import SVN -from MgaRepo.log import UserTagParser -from os.path import basename, dirname, abspath, lexists, join -from os import chdir, getcwd -from tempfile import mkstemp -import sys -import re -import time -from xml.etree import ElementTree -import subprocess - -class GITLogEntry(VCSLogEntry): - def __init__(self, revision, author, date): - VCSLogEntry.__init__(self, revision, author, data) - -class GIT(VCS): - vcs_dirname = ".git" - vcs_name = "git" - def __init__(self, path=None, url=None): - VCS.__init__(self, path, url) - self.vcs_command = config.get("global", "git-command", ["git"]) - self.vcs_supports['clone'] = True - self.env_defaults = {"GIT_SSH": self.vcs_wrapper} - - def configget(self, key="", location="--local"): - cmd = ["config", location, "--get-regexp", key] - config = None - status, output = self._execVcs(*cmd, noerror=True) - if not status and output: - config = eval("{'" + output.replace("\n", "',\n'").replace(" ", "' : '") + "'}") - return config - - def configset(self, config, location="--local"): - cmd = ("config", location) - for pair in config.items(): - status, output = self._execVcs(*cmd + pair) - if status: - return False - return True - - def clone(self, url, targetpath, fullnames=True, **kwargs): - for vcs in (SVN, GIT): - if lexists(join(targetpath, vcs.vcs_dirname)): - raise Error("target path %s already contains %s repository, aborting..." % (targetpath, vcs.vcs_name)) - - self.init(url, targetpath, fullnames=True, **kwargs) - if url.split(':')[0].find("svn") < 0: - return VCS.clone(self, url, **kwargs) - else: - return self.update(targetpath, clone=True, **kwargs) - - def init(self, url, targetpath, fullnames=True, branch=None, **kwargs): - # verify repo url - execcmd("svn", "info", url) - - topurl = dirname(url) - trunk = basename(url) - tags = "releases" - # cloning svn braches as well should rather be optionalif reenabled.. - #cmd = ["svn", "init", topurl, "--trunk="+trunk, "--tags="+tags", targetpath] - - cmd = ["svn", "init", url, abspath(targetpath)] - self._execVcs(*cmd, **kwargs) - os.environ.update({"GIT_WORK_TREE" : abspath(targetpath), "GIT_DIR" : join(abspath(targetpath),".git")}) - - if fullnames: - usermap = UserTagParser() - # store configuration in local git config so that'll be reused later when ie. updating - gitconfig = {"svn-remote.authorlog.url" : usermap.url, - "svn-remote.authorlog.defaultmail": usermap.defaultmail} - self.configset(gitconfig) - - if branch: - execcmd(("git", "init", "-q", self.path), **kwargs) - execcmd(("git", "checkout", "-q", branch), **kwargs) - cmd = ["svn", "rebase", "--local"] - status, output = self._execVcs(*cmd, **kwargs) - - return True - - def info(self, path, **kwargs): - cmd = ["svn", "info", path + '@' if '@' in path else path] - status, output = self._execVcs(local=True, noerror=True, *cmd, **kwargs) - if (("Not a git repository" not in output) and \ - ("Unable to determine upstream SVN information from working tree history" not in output)): - return output.splitlines() - return None - - def status(self, path, **kwargs): - cmd = ["status", "--porcelain", path + '@' if '@' in path else path] - if kwargs.get("verbose"): - cmd.append("-v") - if kwargs.get("noignore"): - cmd.append("--ignored") - if kwargs.get("quiet"): - cmd.append("-uno") - else: - cmd.append("-uall") - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return [(x[0], x[8:]) for x in output.splitlines()] - return None - - def update(self, targetpath, clone=False, **kwargs): - os.environ.update({"GIT_WORK_TREE" : abspath(targetpath), "GIT_DIR" : join(abspath(targetpath),".git")}) - - if not clone: - cmd = ["svn", "log", "--oneline", "--limit=1"] - retval, result = self._execVcs(*cmd) - if retval: - return retval - - revision = result.split() - - if revision[0][0] == 'r': - startrev = "-r"+str(int(revision[0][1:])+1) - else: - startrev = "BASE" - - cmd = ["svn", "propget", "svn:entry:committed-rev"] - retval, lastrev = self._execVcs(*cmd) - if retval: - return retval - - #cmd = ["config", "--get-regexp", '^svn-remote.svn.(url|fetch)'] - cmd = ["config", "--get", "svn-remote.svn.url"] - retval, result = self._execVcs(*cmd) - if retval: - return retval - - #result = result.strip().split() - #url = result[1] + "/" + result[3].split(":")[0] - url = result.strip() - - # To speed things up on huge repositories, we'll just grab all the - # revision numbers for this specific directory and grab these only - # in stead of having to go through each and every revision... - cmd = ["svn", "log", "-g", "--xml", url] - if not clone: - cmd.append("%s:%s" % (startrev,lastrev)) - retval, result = execcmd(*cmd) - if retval: - return retval - - xmllog = ElementTree.fromstring(result) - logentries = xmllog.getiterator("logentry") - revisions = [] - for entry in logentries: - revisions.append(int(entry.attrib["revision"])) - revisions.sort() - - fetchcmd = ["svn", "fetch", "--log-window-size=1000"] - gitconfig = self.configget("svn-remote.authorlog") - if gitconfig: - usermap = UserTagParser(url=gitconfig.get("svn-remote.authorlog.url"),defaultmail=gitconfig.get("svn-remote.authorlog.defaultmail")) - usermapfile = usermap.get_user_map_file() - fetchcmd.extend(("--authors-file", usermapfile)) - fetchcmd.append("") - - while revisions: - fetchcmd[-1] = "-r%d"%revisions.pop(0) - self._execVcs(*fetchcmd, **kwargs) - if gitconfig: - usermap.cleanup() - - cmd = ["svn", "rebase", "--log-window-size=1000", "--local", "--fetch-all", "git-svn"] - status, output = self._execVcs(*cmd, **kwargs) - if status == 0: - return [x.split() for x in output.split()] - return None - - def remote(self, *args, **kwargs): - cmd = ["remote"] + list(args) - status, output = self._execVcs(*cmd, **kwargs) - return status, output - - def pull(self, *args, **kwargs): - cmd = ["pull"] + list(args) - status, output = self._execVcs(*cmd, **kwargs) - return status, output - - def push(self, *args, **kwargs): - cmd = ["push"] + list(args) - status, output = self._execVcs(*cmd, **kwargs) - return status, output - -class GITLook(VCSLook): - def __init__(self, repospath, txn=None, rev=None): - VCSLook.__init__(self, repospath, txn, rev) - -# vim:et:ts=4:sw=4 diff --git a/MgaRepo/layout.py b/MgaRepo/layout.py index 395e996..97f634d 100644 --- a/MgaRepo/layout.py +++ b/MgaRepo/layout.py @@ -5,7 +5,6 @@ import urllib.parse from MgaRepo import Error, config from MgaRepo.svn import SVN -from MgaRepo.vcsutil import detectVCS __all__ = ["package_url", "checkout_url", "repository_url", "get_url_revision"] @@ -133,8 +132,7 @@ def repository_url(mirrored=False): raise Error("you need to set the 'repository' " \ "configuration option on mgarepo.conf") url = convert_default_parent(default_parent) - vcs = detectVCS(url) - return vcs.url + return url def package_url(name_or_url, version=None, release=None, distro=None, backports=None, mirrored=True, obsolete=None): diff --git a/MgaRepo/log.py b/MgaRepo/log.py index 8e062ad..fab7d6e 100644 --- a/MgaRepo/log.py +++ b/MgaRepo/log.py @@ -1,3 +1,4 @@ +#!/usr/bin/python3 from MgaRepo import Error, config, layout from MgaRepo.svn import SVN from MgaRepo.util import execcmd @@ -7,7 +8,6 @@ from io import StringIO import sys import os -import os.path import re import time import locale @@ -27,6 +27,7 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False): Is here where things should be changed if "automatic release increasing" will be used. """ + from MgaRepo.rpmutil import rpm_macros_defs svn = SVN() pkgcurrenturl = os.path.join(pkgdirurl, "current") specurl = os.path.join(pkgcurrenturl, "SPECS") @@ -42,11 +43,17 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False): if not found: raise Error("no .spec file found inside %s" % specurl) specpath = found[0] - options = [("--define", expr) for expr in macros] - command = ["rpm", "-q", "--qf", "%{EPOCH}:%{VERSION}-%{RELEASE}\n", - "--specfile", specpath] - command.extend(options) - status, output = execcmd(*command) + options = rpm_macros_defs(macros) + command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " + "--specfile %s %s") % + (specpath, options)) + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + pipe.wait() + output = pipe.stdout.read().decode('utf8') + error = pipe.stderr.read().decode('utf8') + if pipe.returncode != 0: + raise Error("Error in command %s: %s" % (command, error)) releases = output.split() try: epoch, vr = releases[0].split(":", 1) @@ -74,11 +81,17 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False): if not found: raise Error("no .src.rpm file found inside %s" % srpmurl) srpmpath = found[0] - options = [("--define", expr) for expr in macros] - command = ["rpm", "-q", "--qf", "%{EPOCH}:%{VERSION}-%{RELEASE}\n", - "--specfile", specpath] - command.extend(options) - status, output = execcmd(*command) + options = rpm_macros_defs(macros) + command = (("rpm -qp --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " + " %s %s") % + (srpmpath, options)) + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + pipe.wait() + output = pipe.stdout.read().decode('utf8') + error = pipe.stderr.read().decode('utf8') + if pipe.returncode != 0: + raise Error("Error in command %s: %s" % (command, error)) releases = output.split() try: epoch, vr = releases[0].split(":", 1) @@ -244,12 +257,11 @@ def group_revisions_by_author(currentlog): emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>") -usermap = {} + def get_author_name(author): found = emailpat.match(config.get("users", author, author)) - gold = emailpat.match(usermap.get(author,"")) - name = ((found and found.group("name")) or (gold and gold.group("name")) or author) - email = ((found and found.group("email")) or (gold and gold.group("email")) or author+"@mageia.org") + name = ((found and found.group("name")) or author) + email = ((found and found.group("email")) or author) return name, email def parse_raw_date(rawdate): @@ -342,18 +354,15 @@ def dump_file(releases, currentlog=None, template=None): first = False else: draft = draft + spaces + line + "\n" - if rel is not releases_author[-1]: - draft += "\n" + draft += '\n' else: # default template if not releases_author[-1].visible: releases_author = releases_author[:-1] for rel in releases_author: if not rel.released: - unreleased = " (not released yet)\n" - else: - unreleased = "" - draft = draft + "* {0} {1} <{2}> {3}-{4}\n{5}+ Revision: {6}\n".format(rel.date, rel.author_name, rel.author_email, rel.version, rel.release, unreleased, rel.revision) + draft = " (not released yet)\n" + draft = draft + "* {0} {1} <{2}> {3}-{4}\n+ Revision: {5}\n".format(rel.date, rel.author_name, rel.author_email, rel.version, rel.release, rel.revision) if not rel.visible: draft = draft + "+ rebuild (emptylog)\n" for rev in rel.release_revisions: @@ -366,8 +375,7 @@ def dump_file(releases, currentlog=None, template=None): for rev in author.revisions: for line in rev.lines: draft = draft + line + "\n" - if rel is not releases_author[-1]: - draft += "\n" + draft += "\n" return draft class InvalidEntryError(Exception): @@ -570,14 +578,14 @@ def get_old_log(pkgdirurl): chlog = StringIO() oldurl = config.get("log", "oldurl") if oldurl: - svn = SVN(url=oldurl) + svn = SVN() tmpdir = tempfile.mktemp() try: if oldurl == '.' or oldurl.startswith('./'): pkgoldurl = os.path.join(pkgdirurl, oldurl) else: pkgname = layout.package_name(pkgdirurl) - pkgoldurl = os.path.join(svn.url, pkgname) + pkgoldurl = os.path.join(oldurl, pkgname) try: # we're using HEAD here because fixes in misc/ (oldurl) may # be newer than packages' last changed revision. @@ -598,74 +606,9 @@ def get_old_log(pkgdirurl): chlog.seek(0) return chlog -from html.parser import HTMLParser -from urllib.request import urlopen -class UserTagParser(HTMLParser): - li = False - ahref = False - userpage = None - namepat = re.compile("(?P<name>.*?)\s*\((?P<user>.*?)\)") - usermap = {} - usermapfile = None - - def __init__(self, url=None, defaultmail=None, *cmd, **kwargs): - HTMLParser.__init__(self, *cmd, **kwargs) - self.url = url or "http://people.mageia.org/u/" - self.defaultmail = defaultmail or "mageia.org" - - def handle_starttag(self, tag, attrs): - if tag == "li": - self.li = True - if self.li and tag == "a": - for att in attrs: - if att[0] == "href": - self.ahref = True - self.userpage = att[1] - - def handle_endtag(self, tag): - if self.li and tag == "a": - self.ahref = False - self.userpage = None - if tag == "li": - self.li = False - - def handle_data(self, data): - if self.li and self.ahref: - found = self.namepat.match(data) - if found: - user = found.group("user") - name = found.group("name") - if user and name and user+".html" == self.userpage: - self.usermap[user] = "%s <%s@%s>" % (name, user, self.defaultmail) - - def get_user_map(self): - f = urlopen(self.url) - userhtml = f.read().decode("UTF-8") - f.close() - self.feed(userhtml) - return self.usermap - - def get_user_map_file(self): - if not self.usermap: - self.get_user_map() - self.usermapfile = tempfile.mkstemp(suffix=".txt", prefix="usermap") - f = open(self.usermapfile[0], "w", encoding="UTF-8") - f.writelines("%s = %s\n" % user for user in sorted(self.usermap.items())) - f.close() - return self.usermapfile[1] - - def cleanup(self): - if os.path.exists(self.usermapfile[1]): - os.unlink(self.usermapfile[1]) - -def _map_user_names(): - if not usermap: - parser = UserTagParser() - usermap.update(parser.get_user_map()) - def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, submit=False, sort=False, template=None, macros=[], exported=None, - oldlog=False, create=False, fullnames=False): + oldlog=False, create=False): """Generates the changelog for a given package URL @another: a stream with the contents of a changelog to be merged with @@ -691,9 +634,6 @@ def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, """ newlog = StringIO() if svn: - if fullnames: - if not usermap: - _map_user_names() rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, template=template, macros=macros, exported=exported, create=create) newlog.write(rawsvnlog) @@ -708,21 +648,19 @@ def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, return newlog def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, - submit=False, sort=False, template=None, macros=[], exported=None, create=False, fullnames=False): + submit=False, sort=False, template=None, macros=[], exported=None, create=False): with open(specfile, encoding = 'utf-8') as fi: spec, oldchlog = split_spec_changelog(fi) another = None if config.getbool("log", "merge-spec", False): another = oldchlog sort = sort or config.getbool("log", "sort", False) - if fullnames: - _map_user_names() chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size, submit=submit, sort=sort, template=template, macros=macros, exported=exported, oldlog=True, create=create) with open(specfile, "w", encoding='utf-8') as fo: fo.writelines(spec) - fo.write("\n%changelog\n") + fo.write("\n\n%changelog\n") fo.writelines(chlog) if __name__ == "__main__": diff --git a/MgaRepo/rpmutil.py b/MgaRepo/rpmutil.py index 78e086d..769167c 100644 --- a/MgaRepo/rpmutil.py +++ b/MgaRepo/rpmutil.py @@ -1,10 +1,9 @@ +#!/usr/bin/python3 from MgaRepo import Error, config from MgaRepo import mirror, layout, log, binrepo -from MgaRepo.simplerpm import SRPM -from MgaRepo.util import execcmd, CommandError -from MgaRepo.git import GIT from MgaRepo.svn import SVN -from MgaRepo.vcsutil import detectVCS +from MgaRepo.simplerpm import SRPM +from MgaRepo.util import execcmd from MgaRepo.command import default_parent import rpm import urllib.parse @@ -16,12 +15,12 @@ import sys import os def get_spec(pkgdirurl, targetdir=".", submit=False): - svn = detectVCS(pkgdirurl) + svn = SVN() tmpdir = tempfile.mktemp() try: geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") mirror.info(geturl) - svn.export(geturl, tmpdir) + svn.export("'%s'" % geturl, tmpdir) speclist = glob.glob(os.path.join(tmpdir, "*.spec")) if not speclist: raise Error("no spec files found") @@ -34,9 +33,14 @@ def get_spec(pkgdirurl, targetdir=".", submit=False): if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) +def rpm_macros_defs(macros): + defs = ("--define \"%s %s\"" % macro for macro in macros) + args = " ".join(defs) + return args + #FIXME move it to another module def rev_touched_url(url, rev): - svn = detectVCS(url) + svn = SVN() info = svn.info2(url) if info is None: raise Error("can't fetch svn info about the URL: %s" % url) @@ -91,21 +95,16 @@ def get_srpm(pkgdirurl, template = None, macros = [], verbose = 0, - strict = False, - fullnames = False): - svn = detectVCS(pkgdirurl) + strict = False): + svn = SVN() tmpdir = tempfile.mktemp() - topdir = "_topdir %s" % tmpdir - builddir = "_builddir %s/%s" % (tmpdir, "BUILD") - rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS") - sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES") - specdir = "_specdir %s/%s" % (tmpdir, "SPECS") - srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS") - patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES") - temppath = "_tmppath %s" % (tmpdir) - - rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir, - sourcedir, specdir, srcrpmdir, patchdir, temppath)] + topdir = "--define '_topdir %s'" % tmpdir + builddir = "--define '_builddir %s/%s'" % (tmpdir, "BUILD") + rpmdir = "--define '_rpmdir %s/%s'" % (tmpdir, "RPMS") + sourcedir = "--define '_sourcedir %s/%s'" % (tmpdir, "SOURCES") + specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS") + srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS") + patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES") try: if mode == "version": @@ -134,29 +133,24 @@ def get_srpm(pkgdirurl, if config.getbool("srpm", "run-prep", False): makefile = os.path.join(tmpdir, "Makefile") if os.path.exists(makefile): - execcmd(("make", "-C", tmpdir, "srpm-prep")) + execcmd("make", "-C", tmpdir, "srpm-prep") if not speclist: raise Error("no spec files found") spec = speclist[0] + defs = rpm_macros_defs(macros) sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") if packager: packager = " --define 'packager %s'" % packager - sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") - args = [sourcecmd, "-bs", "--nodeps"] - for pair in rpmdefs: - args.extend(pair) - for pair in macros: - args.extend(("--define", "%s %s" % pair)) - args.append(spec) if svnlog: submit = not not revision try: log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, - template=template, macros=macros, exported=tmpdir, fullnames=fullnames) + template=template, macros=macros, exported=tmpdir) except: - #cmd = [sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir - execcmd(args) + execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % + (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir, + srcrpmdir, patchdir, packager, spec, defs)) cp_srpms(revision, revname, geturl, targetdirs, srpmsdir, verbose) log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir, create=True) @@ -168,15 +162,9 @@ def get_srpm(pkgdirurl, if status != 0: raise Error("script %s failed" % script) - try: - execcmd(args) - except CommandError as e: - if config.getbool("global", "verbose"): - cmdline = e.cmdline + "\n" - else: - cmdline = "" - raise Error("error while creating the source RPM " - "(with %s):\n%s%s" % (sourcecmd, cmdline, e.output)) + execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % + (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir, + srcrpmdir, patchdir, packager, spec, defs)) # copy the generated SRPMs to their target locations targetsrpms = cp_srpms(revision, revname, geturl, targetdirs, srpmsdir, verbose) @@ -187,7 +175,7 @@ def get_srpm(pkgdirurl, def patch_spec(pkgdirurl, patchfile, log=""): #FIXME use get_spec - svn = detectVCS(pkgdirurl) + svn = SVN() tmpdir = tempfile.mktemp() try: geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") @@ -196,7 +184,7 @@ def patch_spec(pkgdirurl, patchfile, log=""): if not speclist: raise Error("no spec files found") spec = speclist[0] - status, output = execcmd(["patch", spec, patchfile]) + status, output = execcmd("patch", spec, patchfile) if status != 0: raise Error("can't apply patch:\n%s\n" % output) else: @@ -207,6 +195,7 @@ def patch_spec(pkgdirurl, patchfile, log=""): def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, baseurl=None, baseold=None, logmsg=None, rename=True): + svn = SVN() srpm = SRPM(srpmfile) tmpdir = tempfile.mktemp() if baseurl: @@ -214,7 +203,6 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, else: pkgurl = layout.package_url(srpm.name, distro=branch, mirrored=False) - svn = detectVCS(pkgurl) print("Importing package to %s" % pkgurl) try: if srpm.epoch: @@ -276,9 +264,9 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, svn.remove(entrypath) # Copy all files - execcmd(["cp", "-rf", uspecsdir, currentdir]) + execcmd("cp -rlf", uspecsdir, currentdir) if os.path.isdir(usourcesdir): - execcmd(["cp", "-rlf", usourcesdir, currentdir]) + execcmd("cp -rlf", usourcesdir, currentdir) # Add new entries for entry in [x for x in uspecsentries @@ -367,106 +355,8 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, log="Copying release %s-%s to releases/ directory." % (version, srpm.release)) -def build_rpm(build_cmd="b", - verbose=True, - rpmlint=True, - short_circuit=False, - packager = None, - installdeps = True, - use_dnf = False, - svnlog = False, - fullnames = True, - macros = [], - **rpmargs): - top = os.getcwd() - topdir = "_topdir %s" % top - builddir = "_builddir %s/%s" % (top, "BUILD") - rpmdir = "_rpmdir %s/%s" % (top, "RPMS") - sourcedir = "_sourcedir %s/%s" % (top, "SOURCES") - specdir = "_specdir %s/%s" % (top, "SPECS") - srcrpmdir = "_srcrpmdir %s/%s" % (top, "SRPMS") - patchdir = "_patchdir %s/%s" % (top, "SOURCES") - - build = os.path.join(top, "BUILD") - if not os.path.exists(build): - os.mkdir(build) - specsdir = os.path.join(top, "SPECS") - speclist = glob.glob(os.path.join(specsdir, "*.spec")) - if not speclist: - raise Error("no spec files found") - spec = speclist[0] - - # If we're building package with %changelog, we'll make a temporary - # copy of the spec file with %changelog applied that we'll use for - # building. This way we avoid modifying files in repository. - # TODO: implement support for external changelog in rpm - if svnlog: - vcs = detectVCS(top) - specsdir = tempfile.mkdtemp() - shutil.copy(spec, specsdir) - specdir = "_specdir "+specsdir - spec = os.path.join(specsdir,os.path.basename(spec)) - info = vcs.info2(top) - pkgdirurl = layout.remove_current(info["URL"]) - log.specfile_svn2rpm(pkgdirurl, spec, rev=None, submit=False, - template=None, macros=macros, exported=top, fullnames=fullnames) - - rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir, - sourcedir, specdir, srcrpmdir, patchdir)] - - if packager: - rpmdefs.append(("--define", "packager %s" % packager)) - - if rpmlint: - rpmdefs.append(("--define", "_build_pkgcheck_set %{_bindir}/rpmlint")) - - rpmbuild = config.get("helper", "rpmbuild", "rpmbuild") - args = [rpmbuild, spec] - if short_circuit: - args.append("--short-circuit") - for pair in rpmdefs: - args.extend(pair) - for pair in macros: - args.extend(("--define", "%s %s" % pair)) - args.extend(("--define", "_disable_source_fetch 0")) - args.extend(*rpmargs.values()) - os.environ["LC_ALL"] = "C" - # First check whether dependencies are satisfied - status, output = execcmd(*args + ["--nobuild"], show=verbose, collecterr=True, noerror=True) - if status: - if "error: Failed build dependencies:" in output: - if not installdeps: - raise Error("Automatic installation of dependencies disabled," - "aborting...") - else: - if verbose: - print("Installing missing build dependencies") - if use_dnf: - pkg_mgr_base = ["dnf"] - pkg_mgr_builddep = pkg_mgr_base + ["--assume-yes", "--setopt=install_weak_deps=False", "builddep"] - else: - pkg_mgr_base = ["urpmi"] - pkg_mgr_builddep = pkg_mgr_base + ["--auto", "--buildrequires", "--no-recommends"] - if os.getuid() != 0: - print("Trying to obtain privileges for installing build dependencies:") - sudocheck = ["sudo", "-l"] + pkg_mgr_base - status, output = execcmd(*sudocheck, collecter=True, noerror=True) - if status: - raise Error("%s\nFailed! Cannot proceed without, aborting..." - % output.splitlines()[-1]) - cmd_base = ["sudo"] + pkg_mgr_builddep - else: - cmd_base = pkg_mgr_builddep - cmd = cmd_base + [spec] - status, output = execcmd(*cmd, show=verbose, collecter=True, noerror=True) - - status, output = execcmd(*args + ["-b"+build_cmd], show=verbose) - if svnlog: - if os.path.isdir(specsdir): - shutil.rmtree(specsdir) - def create_package(pkgdirurl, log="", verbose=0): - svn = detectVCS(pkgdirurl) + svn = SVN() tmpdir = tempfile.mktemp() try: basename = layout.package_name(pkgdirurl) @@ -507,7 +397,7 @@ revision: %s return log def mark_release(pkgdirurl, version, release, revision): - svn = detectVCS(pkgdirurl) + svn = SVN() releasesurl = layout.checkout_url(pkgdirurl, releases=True) versionurl = "/".join([releasesurl, version]) releaseurl = "/".join([versionurl, release]) @@ -529,7 +419,7 @@ def mark_release(pkgdirurl, version, release, revision): log=markreleaselog) def check_changed(pkgdirurl, all=0, show=0, verbose=0): - svn = detectVCS(pkgdirurl) + svn = SVN() if all: baseurl = pkgdirurl packages = [] @@ -600,48 +490,33 @@ def checkout(pkgdirurl, path=None, revision=None, branch=None, distro=None, back if path is None: path = layout.package_name(pkgdirurl) mirror.info(current, write=True) - svn = detectVCS(pkgdirurl) + svn = SVN() svn.checkout(current, path, rev=revision, show=1) if not spec: binrepo.download_binaries(path) -def clone(pkgdirurl, path=None, revision=None, branch=None, distro=None, backports=None, - spec=False, fullnames = True, bindownload = True): - o_pkgdirurl = pkgdirurl - pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro, backports=backports) - append = None - if spec: - append = "SPECS" - current = layout.checkout_url(pkgdirurl, branch=branch, backports=backports, - append_path=append) - if path is None: - path = layout.package_name(pkgdirurl) - mirror.info(current, write=True) - git = GIT() - git.clone(current, path, fullnames=fullnames, show=1) - if not spec and bindownload: - binrepo.download_binaries(path) - -def getpkgtopdir(basedir=os.path.curdir): - vcs = detectVCS(basedir) - if vcs: - basedir = os.path.relpath(vcs.get_topdir()) - if ispkgtopdir(basedir, vcs_dirname=vcs.vcs_dirname): - return basedir - raise Error("can't find top package directories SOURCES and SPECS") - -def ispkgtopdir(path=None, vcs_dirname=None): +def getpkgtopdir(basedir=None): + #FIXME this implementation doesn't work well with relative path names, + # which is something we need in order to have a friendlier output + if basedir is None: + basedir = os.path.curdir + while not ispkgtopdir(basedir): + if os.path.abspath(basedir) == "/": + raise Error("can't find top package directories SOURCES and SPECS") + basedir = os.path.join(basedir, os.path.pardir) + if basedir.startswith("./"): + basedir = basedir[2:] + return basedir + +def ispkgtopdir(path=None): if path is None: path = os.getcwd() names = os.listdir(path) - if not vcs_dirname: - vcs = detectVCS(path) - vcs_dirname = vcs.vcs_dirname - return (vcs_dirname in names and "SPECS" in names and "SOURCES" in names) + return (".svn" in names and "SPECS" in names and "SOURCES" in names) def sync(dryrun=False, commit=False, download=False): + svn = SVN() topdir = getpkgtopdir() - svn = detectVCS(topdir) spath = binrepo.sources_path(topdir) binrepoentries = binrepo.parse_sources(spath) # run svn info because svn st does not complain when topdir is not an @@ -723,7 +598,7 @@ def sync(dryrun=False, commit=False, download=False): upload([path], commit=commit) def commit(target=".", message=None, logfile=None): - svn = detectVCS(target) + svn = SVN() status = svn.status(target, quiet=True) if not status: print("nothing to commit") @@ -758,41 +633,39 @@ def spec_sources(topdir): return sources def update(target=None): - vcs = None + svn = SVN() info = None - vcs_target = None + svn_target = None br_target = None if target: - vcs_target = target + svn_target = target else: top = getpkgtopdir() - vcs_target = top + svn_target = top br_target = top - if vcs_target: - vcs = detectVCS(vcs_target) - vcs.update(vcs_target, show=True) + if svn_target: + svn.update(svn_target, show=True) if br_target: - if not vcs: - vcs = detectVCS(br_target) - info = vcs.info2(vcs_target) - if not br_target and not vcs_target: - raise Error("target not in %s nor in binaries "\ - "repository: %s" % (type(vcs).__name__,target)) + info = svn.info2(svn_target) + if not br_target and not svn_target: + raise Error("target not in SVN nor in binaries "\ + "repository: %s" % target) url = info["URL"] binrepo.download_binaries(br_target) def upload(paths, commit=False): - topdir = getpkgtopdir() - svn = detectVCS(topdir) for path in paths: if os.path.isdir(path) or binrepo.is_binary(path): + topdir = getpkgtopdir() binrepo.upload_binary(topdir, os.path.basename(path)) binrepo.update_sources(topdir, added=[path]) if commit: + svn = SVN() silent = config.get("log", "ignore-string", "SILENT") message = "%s: new file %s" % (silent, path) svn.commit(binrepo.sources_path(topdir), log=message) else: + svn = SVN() svn.add(path, local=True) if commit: silent = config.get("log", "ignore-string", "SILENT") @@ -801,15 +674,16 @@ def upload(paths, commit=False): def delete(paths, commit=False): silent = config.get("log", "ignore-string", "SILENT") - topdir = getpkgtopdir() - svn = detectVCS(topdir) for path in paths: message = "%s: delete file %s" % (silent, path) if binrepo.is_binary(path): + topdir = getpkgtopdir() binrepo.update_sources(topdir, removed=[os.path.basename(path)]) if commit: + svn = SVN() svn.commit(binrepo.sources_path(topdir), log=message) else: + svn = SVN() svn.remove(path, local=True) if commit: svn.commit(path, log=message) @@ -818,7 +692,7 @@ def obsolete(pkgdirurl, branch=None, distro=None, backports=None, commit=False, o_pkgdirurl = pkgdirurl pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro, backports=backports) pkgdest = layout.package_url(o_pkgdirurl, obsolete=True, backports=backports) - svn = detectVCS(pkgdirurl) + svn = SVN() svn.mv(pkgdirurl, pkgdest, message=log) if commit: svn.commit(path, log=log) @@ -853,7 +727,7 @@ def get_submit_info(path): if not os.path.isdir(os.path.join(path, ".svn")): raise Error("subversion directory not found") - svn = detectVCS(path) + svn = SVN() # Now, extract the package name. info = svn.info2(path) @@ -892,25 +766,4 @@ def get_submit_info(path): return name, url, max -def get_pkg_tag(tag, path=os.path.curdir, subpkg=None): - topdir = getpkgtopdir(path) - speclist = glob.glob(os.path.join(topdir, "SPECS", "*.spec")) - if not speclist: - raise Error("no spec files found") - specfile = speclist[0] - - pkg = rpm.spec(specfile) - if subpkg is None: - header = pkg.sourceHeader - elif isinstance(subpkg,int): - header = pkg.packages(subpkg) - else: - raise Error("Subpkg must be the index number of a package,"\ - "or None for source package") - - if isinstance(header[tag],bytes): - return header[tag].decode("utf8") - else: - return header[tag] - # vim:et:ts=4:sw=4 diff --git a/MgaRepo/simplerpm.py b/MgaRepo/simplerpm.py index f248317..a06e694 100644 --- a/MgaRepo/simplerpm.py +++ b/MgaRepo/simplerpm.py @@ -1,3 +1,4 @@ +#!/usr/bin/python3 from MgaRepo.util import execcmd class SRPM: @@ -6,19 +7,15 @@ class SRPM: self._getinfo() def _getinfo(self): - args = ["rpm", "-qp", "--qf", "%{name} %{epoch} %{release} %{version}", - self.filename] - status, output = execcmd(args) + cmdstr = "rpm -qp --nosignature --qf '%%{name} %%{epoch} %%{release} %%{version}' %s" + status, output = execcmd(cmdstr % self.filename) self.name, self.epoch, self.release, self.version = output.split() if self.epoch == "(none)": self.epoch = None def unpack(self, topdir): - args = ["rpm", "-i", "--nodeps", - "--define", "_sourcedir {0}/SOURCES".format(topdir), - "--define", "_specdir {0}/SPECS".format(topdir), - "--define", "_patchdir {0}/SOURCES".format(topdir), - self.filename] - execcmd(args) + execcmd(("rpm -i --nodeps --define '_sourcedir %s/SOURCES' " + + "--define '_specdir %s/SPECS' --define '_patchdir %s/SOURCES' %s") + % (topdir, topdir, topdir, self.filename)) # vim:et:ts=4:sw=4 diff --git a/MgaRepo/svn.py b/MgaRepo/svn.py index cc1ed1b..74f47a3 100644 --- a/MgaRepo/svn.py +++ b/MgaRepo/svn.py @@ -1,5 +1,5 @@ -from MgaRepo.util import execcmd -from MgaRepo.VCS import * +from MgaRepo import Error, SilentError, config +from MgaRepo.util import execcmd, get_auth import sys import os import re @@ -7,32 +7,445 @@ import time __all__ = ["SVN", "SVNLook", "SVNLogEntry"] -class SVNLogEntry(VCSLogEntry): +class SVNLogEntry: def __init__(self, revision, author, date): - VCSLogEntry.__init__(self, revision, author, data) - -class SVN(VCS): - vcs_dirname = ".svn" - vcs_name = "svn" - def __init__(self, path=None, url=None): - VCS.__init__(self, path, url) - self.vcs_command = config.get("global", "svn-command", ["svn"]) - self.env_defaults = {"SVN_SSH": self.vcs_wrapper} - - def drop_ssh_if_no_auth(self, url): - if url and url.startswith("svn+ssh://"): - cmd = ["info", "--non-interactive", "--no-newline", "--show-item", "url", url] - status, output = self._execVcs(*cmd, local=True, noerror=True, show=False) - if status == 1 and (("E170013" in output) or ("E210002" in output)): - url = url.replace("svn+ssh://", "svn://") - status, output = self._execVcs(*cmd, local=True, noerror=True, show=False) - if status == 0 and output == url: - pass - return url - -class SVNLook(VCSLook): + self.revision = revision + self.author = author + self.date = date + self.changed = [] + self.lines = [] + + def __lt__(self, other): + return (self.date < other.date) + + def __eq__(self,other): + return (self.date == other.date) + +class SVN: + def _execsvn(self, *args, **kwargs): + localcmds = ("add", "revert", "cleanup", "mv") + if not kwargs.get("show") and args[0] not in localcmds: + args = list(args) + args.append("--non-interactive") + else: + if args[0] == "mv": + kwargs["geterr"] = False + else: + kwargs["geterr"] = True + kwargs["cleanerr"] = True + if kwargs.get("xml"): + args.append("--xml") + self._set_env() + svn_command = config.get("global", "svn-command", "svn") + cmdstr = svn_command + " " + " ".join(args) + try: + if args[0] in ('info', 'checkout','log'): + kwargs['info'] = True + else: + kwargs['info'] = False + return execcmd(cmdstr, **kwargs) + except Error as e: + msg = None + if e.args: + if "Permission denied" in e.args: + msg = ("It seems ssh-agent or ForwardAgent are not setup " + "or your username is wrong. See " + "https://wiki.mageia.org/en/Packagers_ssh" + " for more information.") + elif "authorization failed" in e.args: + msg = ("Note that mgarepo does not support any HTTP " + "authenticated access.") + if kwargs.get("show") and \ + not config.getbool("global", "verbose", 0): + # svn has already dumped error messages, we don't need to + # do it too + if msg: + sys.stderr.write("\n") + sys.stderr.write(msg) + sys.stderr.write("\n") + raise SilentError + elif msg: + raise Error("%s\n%s" % (e, msg)) + raise + + def _set_env(self): + wrapper = "mgarepo-ssh" + repsys = config.get("global", "mgarepo-cmd") + if repsys: + dir = os.path.dirname(repsys) + path = os.path.join(dir, wrapper) + if os.path.exists(path): + wrapper = path + defaults = {"SVN_SSH": wrapper} + os.environ.update(defaults) + raw = config.get("global", "svn-env") + if raw: + for line in raw.split("\n"): + env = line.strip() + if not env: + continue + try: + name, value = env.split("=", 1) + except ValueError: + sys.stderr.write("invalid svn environment line: %r\n" % env) + continue + os.environ[name] = value + + def _execsvn_success(self, *args, **kwargs): + status, output = self._execsvn(*args, **kwargs) + return status == 0 + + def _add_log(self, cmd_args, received_kwargs, optional=0): + if (not optional or + "log" in received_kwargs or + "logfile" in received_kwargs): + ret = received_kwargs.get("log") + if ret is not None: + cmd_args.append("-m '%s'" % ret) + ret = received_kwargs.get("logfile") + if ret is not None: + cmd_args.append("-F '%s'" % ret) + + def _add_revision(self, cmd_args, received_kwargs, optional=0): + if not optional or "rev" in received_kwargs: + ret = received_kwargs.get("rev") + if isinstance(ret, str): + if not ret.startswith("{"): # if not a datespec + try: + ret = int(ret) + except ValueError: + raise Error("invalid revision provided") + if ret: + cmd_args.append("-r '%s'" % ret) + + def add(self, path, **kwargs): + cmd = ["add", path + '@' if '@' in path else path] + return self._execsvn_success(noauth=1, *cmd, **kwargs) + + def copy(self, pathfrom, pathto, **kwargs): + cmd = ["copy", pathfrom + '@' if '@' in pathfrom else pathfrom, pathto + '@' if '@' in pathto else pathto] + self._add_revision(cmd, kwargs, optional=1) + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def remove(self, path, force=0, **kwargs): + cmd = ["remove", path + '@' if '@' in path else path] + self._add_log(cmd, kwargs) + if force: + cmd.append("--force") + return self._execsvn_success(*cmd, **kwargs) + + def mkdir(self, path, **kwargs): + cmd = ["mkdir", path + '@' if '@' in path else path] + if kwargs.get("parents"): + cmd.append("--parents") + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def _execsvn_commit(self, *cmd, **kwargs): + status, output = self._execsvn(*cmd, **kwargs) + match = re.search("Committed revision (?P<rev>\\d+)\\.$", output) + if match: + rawrev = match.group("rev") + return int(rawrev) + + def commit(self, path, **kwargs): + cmd = ["commit", path + '@' if '@' in path else path] + if kwargs.get("nonrecursive"): + cmd.append("-N") + self._add_log(cmd, kwargs) + return self._execsvn_commit(*cmd, **kwargs) + + def import_(self, path, url, **kwargs): + cmd = ["import", "'%s'" % path, "'%s'" % url] + self._add_log(cmd, kwargs) + return self._execsvn_commit(*cmd, **kwargs) + + def export(self, url, targetpath, **kwargs): + cmd = ["export", "'%s'" % url, targetpath] + self._add_revision(cmd, kwargs, optional=1) + return self._execsvn_success(*cmd, **kwargs) + + def checkout(self, url, targetpath, **kwargs): + cmd = ["checkout", "'%s'" % url, targetpath] + self._add_revision(cmd, kwargs, optional=1) + return self._execsvn_success(*cmd, **kwargs) + + def propget(self, propname, targets, **kwargs): + cmd = ["propget", propname, targets] + if kwargs.get("revprop"): + cmd.append("--revprop") + self._add_revision(cmd, kwargs) + status, output = self._execsvn(local=True, *cmd, **kwargs) + return output + + def propset(self, propname, value, targets, **kwargs): + cmd = ["propset", propname, "'%s'" % value, targets] + return self._execsvn_success(*cmd, **kwargs) + + def propedit(self, propname, target, **kwargs): + cmd = ["propedit", propname, target] + if kwargs.get("rev"): + cmd.append("--revprop") + self._add_revision(cmd, kwargs) + return self._execsvn_success(local=True, show=True, *cmd, **kwargs) + + def revision(self, path, **kwargs): + cmd = ["info", path + '@' if '@' in path else path] + status, output = self._execsvn(local=True, *cmd, **kwargs) + if status == 0: + for line in output.splitlines(): + if line.startswith("Last Changed Rev: "): + return int(line.split()[3]) + return None + + def info(self, path, **kwargs): + cmd = ["info", path + '@' if '@' in path else path] + status, output = self._execsvn(local=True, noerror=True, *cmd, **kwargs) + if (("Not a versioned resource" not in output) and ("svn: warning: W155010" not in output)): + return output.splitlines() + return None + + def info2(self, *args, **kwargs): + lines = self.info(*args, **kwargs) + if lines is None: + return None + pairs = [[w.strip() for w in line.split(":", 1)] for line in lines] + info = {} + for pair in pairs: + if pair != ['']: + info[pair[0]]=pair[1] + return info + + def ls(self, path, **kwargs): + cmd = ["ls", path + '@' if '@' in path else path] + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output.split() + return None + + def status(self, path, **kwargs): + cmd = ["status", path + '@' if '@' in path else path] + if kwargs.get("verbose"): + cmd.append("-v") + if kwargs.get("noignore"): + cmd.append("--no-ignore") + if kwargs.get("quiet"): + cmd.append("--quiet") + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [(x[0], x[8:]) for x in output.splitlines()] + return None + + def cleanup(self, path, **kwargs): + cmd = ["cleanup", path + '@' if '@' in path else path] + return self._execsvn_success(*cmd, **kwargs) + + def revert(self, path, **kwargs): + cmd = ["revert", path + '@' if '@' in path else path] + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def switch(self, url, oldurl=None, path=None, relocate=False, **kwargs): + cmd = ["switch"] + if relocate: + if oldurl is None: + raise Error("You must supply the old URL when "\ + "relocating working copies") + cmd.append("--relocate") + cmd.append(oldurl) + cmd.append(url) + if path is not None: + cmd.append(path) + return self._execsvn_success(*cmd, **kwargs) + + def update(self, path, **kwargs): + cmd = ["update", path + '@' if '@' in path else path] + self._add_revision(cmd, kwargs, optional=1) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def merge(self, url1, url2=None, rev1=None, rev2=None, path=None, + **kwargs): + cmd = ["merge"] + if rev1 and rev2 and not url2: + cmd.append("-r") + cmd.append("%s:%s" % (rev1, rev2)) + cmd.append(url1) + else: + if not url2: + raise ValueError("url2 needed if two revisions are not provided") + if rev1: + cmd.append("%s@%s" % (url1, rev1)) + else: + cmd.append(url1) + if rev2: + cmd.append("%s@%s" % (url2, rev2)) + else: + cmd.append(url2) + if path: + cmd.append(path) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def diff(self, pathurl1, pathurl2=None, **kwargs): + cmd = ["diff", pathurl1] + self._add_revision(cmd, kwargs, optional=1) + if pathurl2: + cmd.append(pathurl2) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output + return None + + def cat(self, url, **kwargs): + cmd = ["cat", url] + self._add_revision(cmd, kwargs, optional=1) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output + return None + + def log(self, url, start=None, end=0, limit=None, **kwargs): + cmd = ["log", "-v", url] + if start is not None or end != 0: + if start is not None and type(start) is not type(0): + try: + start = int(start) + except (ValueError, TypeError): + raise Error("invalid log start revision provided") + if type(end) is not type(0): + try: + end = int(end) + except (ValueError, TypeError): + raise Error("invalid log end revision provided") + start = start or "HEAD" + cmd.append("-r %s:%s" % (start, end)) + if limit is not None: + try: + limit = int(limit) + except (ValueError, TypeError): + raise Error("invalid limit number provided") + cmd.append("--limit %d" % limit) + status, output = self._execsvn(*cmd, **kwargs) + if status != 0: + return None + + revheader = re.compile("^r(?P<revision>[0-9]+) \| (?P<author>[^\|]+) \| (?P<date>[^\|]+) \| (?P<lines>[0-9]+) (?:line|lines)$") + changedpat = re.compile(r"^\s+(?P<action>[^\s]+) (?P<path>[^\s]+)(?: \([^\s]+ (?P<from_path>[^:]+)(?:\:(?P<from_rev>[0-9]+))?\))?$") + logseparator = "-"*72 + linesleft = 0 + entry = None + log = [] + appendchanged = 0 + changedheader = 0 + for line in output.splitlines(): + line = line.rstrip() + if changedheader: + appendchanged = 1 + changedheader = 0 + elif appendchanged: + if not line: + appendchanged = 0 + continue + m = changedpat.match(line) + if m: + changed = m.groupdict().copy() + from_rev = changed.get("from_rev") + if from_rev is not None: + try: + changed["from_rev"] = int(from_rev) + except (ValueError, TypeError): + raise Error("invalid revision number in svn log") + entry.changed.append(changed) + elif linesleft == 0: + if line != logseparator: + m = revheader.match(line) + if m: + linesleft = int(m.group("lines")) + timestr = " ".join(m.group("date").split()[:2]) + timetuple = time.strptime(timestr, + "%Y-%m-%d %H:%M:%S") + entry = SVNLogEntry(int(m.group("revision")), + m.group("author"), timetuple) + log.append(entry) + changedheader = 1 + else: + entry.lines.append(line) + linesleft -= 1 + log.sort() + log.reverse() + return log + + def mv(self, path, dest, message=None, **kwargs): + cmd = ["mv", path, dest, ] + if message: + cmd.append("-m '%s'"%message) + else: + kwargs['show'] = True + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + +class SVNLook: def __init__(self, repospath, txn=None, rev=None): - VCSLook.__init__(self, repospath, txn, rev) - self.execcmd = "svnlook" + self.repospath = repospath + self.txn = txn + self.rev = rev + + def _execsvnlook(self, cmd, *args, **kwargs): + execcmd_args = ["svnlook", cmd, self.repospath] + self._add_txnrev(execcmd_args, kwargs) + execcmd_args += args + execcmd_kwargs = {} + keywords = ["show", "noerror"] + for key in keywords: + if key in kwargs: + execcmd_kwargs[key] = kwargs[key] + return execcmd(*execcmd_args, **execcmd_kwargs) + + def _add_txnrev(self, cmd_args, received_kwargs): + if "txn" in received_kwargs: + txn = received_kwargs.get("txn") + if txn is not None: + cmd_args += ["-t", txn] + elif self.txn is not None: + cmd_args += ["-t", self.txn] + if "rev" in received_kwargs: + rev = received_kwargs.get("rev") + if rev is not None: + cmd_args += ["-r", rev] + elif self.rev is not None: + cmd_args += ["-r", self.rev] + + def changed(self, **kwargs): + status, output = self._execsvnlook("changed", **kwargs) + if status != 0: + return None + changes = [] + for line in output.splitlines(): + line = line.rstrip() + if not line: + continue + entry = [None, None, None] + changedata, changeprop, path = None, None, None + if line[0] != "_": + changedata = line[0] + if line[1] != " ": + changeprop = line[1] + path = line[4:] + changes.append((changedata, changeprop, path)) + return changes + + def author(self, **kwargs): + status, output = self._execsvnlook("author", **kwargs) + if status != 0: + return None + return output.strip() # vim:et:ts=4:sw=4 diff --git a/MgaRepo/util.py b/MgaRepo/util.py index dfae85c..538eae8 100644 --- a/MgaRepo/util.py +++ b/MgaRepo/util.py @@ -1,6 +1,7 @@ +#!/usr/bin/python3 + from MgaRepo import Error, config -import shlex import subprocess import getpass import sys @@ -10,89 +11,73 @@ import select from io import StringIO import httplib2 -class CommandError(Error): - def __init__(self, cmdline, status, output): - self.cmdline = cmdline - self.status = status - self.output = output - - def __str__(self): - return "command failed: %s\n%s\n" % (self.cmdline, self.output) +# Our own version of commands' commands_exec(). We have a commands +# module directory, so we can't import Python's standard module + +# Our own version of commands' getstatusoutput(). We have a commands +# module directory, so we can't import Python's standard module +def commands_getstatusoutput(cmd): + """Return (status, output) of executing cmd in a shell.""" + pipe = subprocess.Popen('{ ' + cmd + '; } 2>&1', stdin = subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines = True, shell = True) + of = pipe.stdout.fileno() + text = '' + pipe.stdin.close() + while True: + text += os.read(of,8192).decode('utf8') + status = pipe.poll() + if status is not None or text == '': + break + if text[-1:] == '\n': text = text[:-1] + return status, text def execcmd(*cmd, **kwargs): - assert (kwargs.get("collecterr") and kwargs.get("show")) or not kwargs.get("collecterr"), \ - ("execcmd is implemented to handle collecterr=True only if show=True") - # split command args - if isinstance(cmd[0], str): - if len(cmd) is 1: - cmdargs = shlex.split(cmd[0]) - else: - cmdargs = cmd[:] + cmdstr = " ".join(cmd) + verbose = config.getbool("global", "verbose", 0) + if kwargs.get('info') : + prefix='LANGUAGE=C LC_TIME=C ' else: - cmdargs = cmd[0][:] - - stdout = None - stderr = None - env = {} - env.update(os.environ) - if kwargs.get("info") or not kwargs.get("show") or (kwargs.get("show") and kwargs.get("collecterr")): - if kwargs.get("info"): - env.update({"LANGUAGE": "C", "LC_TIME": "C"}) + prefix='LANG=C LANGUAGE=C LC_ALL=C ' + if verbose: + print(prefix + cmdstr) + if kwargs.get("show"): + if kwargs.get("geterr"): + err = StringIO() + pstdin = kwargs.get("stdin") if kwargs.get("stdin") else None + p = subprocess.Popen(prefix + cmdstr, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=pstdin) + of = p.stdout.fileno() + ef = p.stderr.fileno() + while True: + r,w,x = select.select((of,ef), (), ()) + odata = None + if of in r: + odata = (os.read(of, 8192)).decode('utf8') + sys.stdout.write(odata) + edata = None + if ef in r: + edata = (os.read(ef, 8192)).decode('utf8') + err.write(edata) + sys.stderr.write(edata) + + status = p.poll() + if status is not None and odata == '' and edata == '': + break + output = err.getvalue() else: - env.update({"LANG": "C", "LANGUAGE": "C", "LC_ALL": "C"}) - stdout = subprocess.PIPE - if kwargs.get("collecterr"): - stderr = subprocess.PIPE + status = os.system(cmdstr) + output = "" + else: + status, output = commands_getstatusoutput(prefix + cmdstr) + if status != 0 and not kwargs.get("noerror"): + if kwargs.get("cleanerr") and not verbose: + raise Error(output) else: - stderr = subprocess.STDOUT - - verbose = config.getbool("global", "verbose", 0) + raise Error("command failed: %s\n%s\n" % (cmdstr, output)) if verbose: - print("cmd: " + str(cmd).lstrip("(").rstrip(")").replace("', '", "' '")) - - proc = subprocess.Popen(cmdargs, shell=False, stdout=stdout, stdin=kwargs.get("stdin"), - stderr=stderr, env=env) - - output = "" - - if kwargs.get("show") and kwargs.get("collecterr"): - error = StringIO() - wl = [] - outfd = proc.stdout.fileno() - errfd = proc.stderr.fileno() - rl = [outfd, errfd] - xl = wl - while proc.poll() is None: - mrl, _, _ = select.select(rl, wl, xl, 0.5) - for fd in mrl: - data = os.read(fd, 8192).decode('utf8') - if fd == errfd: - error.write(data) - sys.stderr.write(data) - else: - sys.stdout.write(data) - output = error.getvalue() - else: - # Make sure to avoid buffer getting full. - # Otherwise if ie. using proc.wait() both python - # and the process will just hang - while proc.poll() is None: - if proc.stdout is not None: - output += proc.stdout.read(8192).decode('utf8') - # Make sure that we've emptied the buffer entirely - if proc.stdout is not None: - output += proc.stdout.read().decode('utf8') - - if kwargs.get("strip", True): - output = output.rstrip() - - if (not kwargs.get("noerror")) and proc.returncode != 0: - if kwargs.get("cleanerr"): - msg = output - cmdline = subprocess.list2cmdline(cmdargs) - raise CommandError(cmdline, proc.returncode, output) - - return proc.returncode, output + print(output) + sys.stdout.write(output) + return status, output def get_output_exec(cmdstr): output = StringIO() diff --git a/MgaRepo/vcsutil.py b/MgaRepo/vcsutil.py deleted file mode 100644 index 91adade..0000000 --- a/MgaRepo/vcsutil.py +++ /dev/null @@ -1,29 +0,0 @@ -from MgaRepo import Error -from MgaRepo.git import GIT -from MgaRepo.svn import SVN -import os - -def detectVCS(url): - if ':' in url: - protocol,uri = url.split(":") - if "svn" in protocol: - return SVN(url=url) - elif "git" in protocol: - return GIT(url=url) - elif "http" in protocol: - if uri.endswith(".git"): - return GIT(url=url) - elif "svn" in uri: - return SVN(url=url) - raise Error("Unknown protocol %s for %s" % (protocol, url)) - elif os.path.exists(url) and os.path.isdir(url): - while True: - url = os.path.abspath(url) - for vcs in (SVN, GIT): - vcsdir = os.path.join(url, vcs.vcs_dirname) - if os.path.exists(vcsdir) and os.path.isdir(vcsdir): - return vcs(path=url) - url = os.path.dirname(url) - if url == "/": - break - raise Error("No supported repository found at path: %s" % url) |