diff options
| -rw-r--r-- | MgaRepo/ConfigParser.py | 14 | ||||
| -rw-r--r-- | MgaRepo/binrepo.py | 2 | ||||
| -rw-r--r-- | MgaRepo/cgi/xmlrpcserver.py | 16 | ||||
| -rw-r--r-- | MgaRepo/commands/submit.py | 9 | ||||
| -rw-r--r-- | MgaRepo/log.py | 6 | ||||
| -rw-r--r-- | MgaRepo/mirror.py | 3 | ||||
| -rw-r--r-- | MgaRepo/plugins/__init__.py | 2 | ||||
| -rw-r--r-- | MgaRepo/plugins/ldapusers.py | 15 | ||||
| -rw-r--r-- | MgaRepo/rpmutil.py | 1 | ||||
| -rw-r--r-- | MgaRepo/svn.py | 67 | ||||
| -rw-r--r-- | MgaRepo/util.py | 16 |
11 files changed, 68 insertions, 83 deletions
diff --git a/MgaRepo/ConfigParser.py b/MgaRepo/ConfigParser.py index 0d5624e..7fc32ac 100644 --- a/MgaRepo/ConfigParser.py +++ b/MgaRepo/ConfigParser.py @@ -167,7 +167,7 @@ class ConfigParser: raise NoOptionError(option, section) if raw: return rawval - return self.__interpolate(rawval, d) + return self.__interpolate(rawval, d, option, section) def getall(self, section, option, raw=0, vars=None): option = self.optionxform(option) @@ -184,8 +184,8 @@ class ConfigParser: return values if vars: d.update(vars) - for i in len(values): - values[i] = self.__interpolate(values[i], d) + for i in range(len(values)): + values[i] = self.__interpolate(values[i], d, option, section) return values def walk(self, section, option=None, raw=0, vars=None): @@ -210,10 +210,10 @@ class ConfigParser: for optname, value in options: if not option or optname == option: if not raw: - value = self.__interpolate(value, d) + value = self.__interpolate(value, d, option, section) yield (optname, value) - def __interpolate(self, value, vars): + def __interpolate(self, value, vars, option, section): rawval = value depth = 0 while depth < 10: @@ -233,10 +233,10 @@ class ConfigParser: return conv(self.get(section, option)) def getint(self, section, option): - return self.__get(section, string.atoi, option) + return self.__get(section, int, option) def getfloat(self, section, option): - return self.__get(section, string.atof, option) + return self.__get(section, float, option) def getboolean(self, section, option): states = {'1': 1, 'yes': 1, 'true': 1, 'on': 1, diff --git a/MgaRepo/binrepo.py b/MgaRepo/binrepo.py index a66df09..7c3aca1 100644 --- a/MgaRepo/binrepo.py +++ b/MgaRepo/binrepo.py @@ -146,7 +146,7 @@ def parse_sources(path): try: f = open(path, encoding="utf-8") except IOError: - return [] + return {} for rawline in f: line = rawline.strip() diff --git a/MgaRepo/cgi/xmlrpcserver.py b/MgaRepo/cgi/xmlrpcserver.py index a1b2b73..60ebb85 100644 --- a/MgaRepo/cgi/xmlrpcserver.py +++ b/MgaRepo/cgi/xmlrpcserver.py @@ -91,21 +91,11 @@ def main(): meth = getattr(iface, method) response = (meth(*parms),) except CgiError as e: - msg = str(e) - try: - msg = msg.decode("iso-8859-1") - except UnicodeError: - pass - response = xmlrpc.client.Fault(1, msg) + response = xmlrpc.client.Fault(1, str(e)) except Exception as e: - msg = str(e) - try: - msg = msg.decode("iso-8859-1") - except UnicodeError: - pass - response = xmlrpc.client.Fault(1, msg) + response = xmlrpc.client.Fault(1, str(e)) sys.stdout.write("Content-type: text/xml\n\n") - sys.stdout.write(xmlrpc.client.dumps(response, methodresponse=1)) + sys.stdout.write(xmlrpc.client.dumps(response, methodresponse=True)) # vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/submit.py b/MgaRepo/commands/submit.py index f0d66c5..ea56e60 100644 --- a/MgaRepo/commands/submit.py +++ b/MgaRepo/commands/submit.py @@ -168,12 +168,9 @@ def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): if submithost is None: submithost = config.get("submit", "host") if submithost is None: - # extract the submit host from the svn host - type, rest = urllib.parse.splittype(pkgdirurl) - host, path = urllib.parse.splithost(rest) - user, host = urllib.parse.splituser(host) - submithost, port = urllib.parse.splitport(host) - del type, user, port, path, rest + # extract the submit host from the first svn host + u = urllib.parse.urlparse(urls[0]) + submithost = u.hostname # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory createsrpm = get_helper("create-srpm") diff --git a/MgaRepo/log.py b/MgaRepo/log.py index dca003f..e8b7e4d 100644 --- a/MgaRepo/log.py +++ b/MgaRepo/log.py @@ -41,7 +41,7 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False): if not create: found = glob.glob(os.path.join(topdir, "SPECS", "*.spec")) if not found: - raise Error("no .spec file found inside %s" % specurl) + raise Error("no .spec file found inside %s/SPECS" % topdir) specpath = found[0] command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " "--specfile %s %s --define '_topdir %s'") % @@ -49,7 +49,7 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False): else: found = glob.glob(os.path.join(topdir, "SRPMS", "*.src.rpm")) if not found: - raise Error("no .src.rpm file found inside %s" % srpmurl) + raise Error("no .src.rpm file found inside %s/SRPMS" % topdir) srpmpath = found[0] command = (("rpm -qp --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " " %s %s --define '_topdir %s'") % @@ -92,7 +92,7 @@ class _Revision: def __repr__(self): lines = repr(self.lines)[:30] + "...]" line = "<_Revision %d author=%r date=%r lines=%s>" % \ - (self.revision, self.author, self.date, lines) + (self.revision, self.author_email, self.date, lines) return line diff --git a/MgaRepo/mirror.py b/MgaRepo/mirror.py index 205cc52..fad9133 100644 --- a/MgaRepo/mirror.py +++ b/MgaRepo/mirror.py @@ -25,10 +25,9 @@ def _joinurl(url, relpath): parsed[3], parsed[4], parsed[5])) return newurl - def strip_username(url): parsed = list(urllib.parse.urlparse(url)) - _, parsed[1] = urllib.parse.splituser(parsed[1]) + parsed[1] = parsed[1].rpartition('@')[-1] newurl = urllib.parse.urlunparse(parsed) return newurl diff --git a/MgaRepo/plugins/__init__.py b/MgaRepo/plugins/__init__.py index 82a4eaa..e734239 100644 --- a/MgaRepo/plugins/__init__.py +++ b/MgaRepo/plugins/__init__.py @@ -17,7 +17,7 @@ def load(): [entry]) def list(): - return list(loaded.keys()) + return [*loaded.keys()] def help(name): from MgaRepo import Error diff --git a/MgaRepo/plugins/ldapusers.py b/MgaRepo/plugins/ldapusers.py index 0edc417..5822cf3 100644 --- a/MgaRepo/plugins/ldapusers.py +++ b/MgaRepo/plugins/ldapusers.py @@ -137,13 +137,12 @@ def make_handler(): "'yes' or 'no'" % raw) try: - from ldap3 import Server, Connection, ALL + import ldap3 #import ldap except ImportError: raise Error("LDAP support needs the ldap3 package "\ "to be installed") - else: - from ldap.utils.conv import escape_filter_chars + from ldap3.utils.conv import escape_filter_chars def users_wrapper(section, option=None, default=None, walk=False): global users_cache @@ -158,23 +157,23 @@ def make_handler(): try: #l = ldap.initialize(uri) - l = Connection(uri, auto_bind=True, use_tls=starttls) + l = ldap3.Connection(uri, auto_bind=True, use_tls=starttls) # if starttls: # l.start_tls() #l.start_tls_s() if binddn: l.bind(binddn, bindpw) - except ldap3.LDAPExceptionError as e: + except ldap3.core.exceptions.LDAPExceptionError as e: raise LDAPError(e) try: data = {"username": escape_filter_chars(option)} filter = interpolate("ldap-filterformat", filterformat, data) attrs = used_attributes(format) try: - found = l.search(search_base='o='.basedn, search_filter=filter, - attributes=attrs, search_scope=SUBTREE) + found = l.search(search_base='o='+basedn, search_filter=filter, + attributes=attrs, search_scope=ldap3.SUBTREE) # found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter, # attrlist=attrs) - except ldap3.LDAPError as e: + except ldap3.core.exceptions.LDAPException as e: raise LDAPError(e) if found: dn, entry = found[0] diff --git a/MgaRepo/rpmutil.py b/MgaRepo/rpmutil.py index a99c08a..1cbbd8a 100644 --- a/MgaRepo/rpmutil.py +++ b/MgaRepo/rpmutil.py @@ -707,6 +707,7 @@ def obsolete(pkgdirurl, branch=None, distro=None, backports=None, commit=False, svn = SVN() svn.mv(pkgdirurl, pkgdest, message=log) if commit: + # TODO: what is path supposed to be? svn.commit(path, log=log) def switch(mirrorurl=None): diff --git a/MgaRepo/svn.py b/MgaRepo/svn.py index 1730901..c0d587f 100644 --- a/MgaRepo/svn.py +++ b/MgaRepo/svn.py @@ -1,12 +1,13 @@ from MgaRepo import Error, SilentError, config -from MgaRepo.util import execcmd, get_auth -import sys +from MgaRepo.util import execcmd import os import re +import sys import time __all__ = ["SVN", "SVNLook", "SVNLogEntry"] + class SVNLogEntry: def __init__(self, revision, author, date): self.revision = revision @@ -17,36 +18,29 @@ class SVNLogEntry: def __lt__(self, other): return (self.date < other.date) - - def __eq__(self,other): + + def __eq__(self, other): return (self.date == other.date) + class SVN: def _execsvn(self, *args, **kwargs): - localcmds = ("add", "revert", "cleanup", "mv") + localcmds = frozenset(("add", "revert", "cleanup", "mv")) if not kwargs.get("show") and args[0] not in localcmds: args = list(args) args.append("--non-interactive") else: - if args[0] == "mv": - kwargs["geterr"] = False - else: - kwargs["geterr"] = True + kwargs["geterr"] = args[0] != "mv" kwargs["cleanerr"] = True if kwargs.get("xml"): + args = list(args) args.append("--xml") self._set_env() svn_command = config.get("global", "svn-command", "svn") cmdstr = svn_command + " " + " ".join(args) try: - if args[0] in ('info', 'checkout','log'): - kwargs['info'] = True - else: - kwargs['info'] = False - if args[0] in ('status','propedit'): - kwargs['noprefix'] = True - else: - kwargs['noprefix'] = False + kwargs["info"] = args[0] in ("info", "checkout", "log") + kwargs["noprefix"] = args[0] in ("status", "propedit") return execcmd(cmdstr, **kwargs) except Error as e: msg = None @@ -64,9 +58,7 @@ class SVN: # svn has already dumped error messages, we don't need to # do it too if msg: - sys.stderr.write("\n") - sys.stderr.write(msg) - sys.stderr.write("\n") + sys.stderr.write("\n" + msg + "\n") raise SilentError elif msg: raise Error("%s\n%s" % (e, msg)) @@ -76,8 +68,8 @@ class SVN: wrapper = "mgarepo-ssh" repsys = config.get("global", "mgarepo-cmd") if repsys: - dir = os.path.dirname(repsys) - path = os.path.join(dir, wrapper) + dirname = os.path.dirname(repsys) + path = os.path.join(dirname, wrapper) if os.path.exists(path): wrapper = path defaults = {"SVN_SSH": wrapper} @@ -101,8 +93,7 @@ class SVN: def _add_log(self, cmd_args, received_kwargs, optional=0): if (not optional or - "log" in received_kwargs or - "logfile" in received_kwargs): + "log" in received_kwargs or "logfile" in received_kwargs): ret = received_kwargs.get("log") if ret is not None: cmd_args.append("-m '%s'" % ret) @@ -114,14 +105,14 @@ class SVN: if not optional or "rev" in received_kwargs: ret = received_kwargs.get("rev") if isinstance(ret, str): - if not ret.startswith("{"): # if not a datespec + if not ret.startswith("{"): # if not a datespec try: ret = int(ret) except ValueError: raise Error("invalid revision provided") if ret: cmd_args.append("-r '%s'" % ret) - + def add(self, path, **kwargs): cmd = ["add", path + '@' if '@' in path else path] return self._execsvn_success(noauth=1, *cmd, **kwargs) @@ -182,7 +173,7 @@ class SVN: self._add_revision(cmd, kwargs) status, output = self._execsvn(local=True, *cmd, **kwargs) return output - + def propset(self, propname, value, targets, **kwargs): cmd = ["propset", propname, "'%s'" % value, targets] return self._execsvn_success(*cmd, **kwargs) @@ -202,7 +193,7 @@ class SVN: if line.startswith("Last Changed Rev: "): return int(line.split()[3]) return None - + def info(self, path, **kwargs): cmd = ["info", path + '@' if '@' in path else path] status, output = self._execsvn(local=True, noerror=True, *cmd, **kwargs) @@ -218,9 +209,9 @@ class SVN: info = {} for pair in pairs: if pair != ['']: - info[pair[0]]=pair[1] + info[pair[0]] = pair[1] return info - + def ls(self, path, **kwargs): cmd = ["ls", path + '@' if '@' in path else path] status, output = self._execsvn(*cmd, **kwargs) @@ -256,8 +247,8 @@ class SVN: cmd = ["switch"] if relocate: if oldurl is None: - raise Error("You must supply the old URL when "\ - "relocating working copies") + raise Error("You must supply the old URL when " + "relocating working copies") cmd.append("--relocate") cmd.append(oldurl) cmd.append(url) @@ -273,8 +264,8 @@ class SVN: return [x.split() for x in output.split()] return None - def merge(self, url1, url2=None, rev1=None, rev2=None, path=None, - **kwargs): + def merge(self, url1, url2=None, rev1=None, rev2=None, path=None, + **kwargs): cmd = ["merge"] if rev1 and rev2 and not url2: cmd.append("-r") @@ -341,7 +332,7 @@ class SVN: if status != 0: return None - revheader = re.compile("^r(?P<revision>[0-9]+) \| (?P<author>[^\|]+) \| (?P<date>[^\|]+) \| (?P<lines>[0-9]+) (?:line|lines)$") + revheader = re.compile(r"^r(?P<revision>[0-9]+) \| (?P<author>[^\|]+) \| (?P<date>[^\|]+) \| (?P<lines>[0-9]+) (?:line|lines)$") changedpat = re.compile(r"^\s+(?P<action>[^\s]+) (?P<path>[^\s]+)(?: \([^\s]+ (?P<from_path>[^:]+)(?:\:(?P<from_rev>[0-9]+))?\))?$") logseparator = "-"*72 linesleft = 0 @@ -388,14 +379,15 @@ class SVN: return log def mv(self, path, dest, message=None, **kwargs): - cmd = ["mv", path, dest, ] + cmd = ["mv", path, dest] if message: - cmd.append("-m '%s'"%message) + cmd.append("-m '%s'" % message) else: kwargs['show'] = True self._add_log(cmd, kwargs) return self._execsvn_success(*cmd, **kwargs) + class SVNLook: def __init__(self, repospath, txn=None, rev=None): self.repospath = repospath @@ -436,7 +428,6 @@ class SVNLook: line = line.rstrip() if not line: continue - entry = [None, None, None] changedata, changeprop, path = None, None, None if line[0] != "_": changedata = line[0] diff --git a/MgaRepo/util.py b/MgaRepo/util.py index 519c199..8435ec2 100644 --- a/MgaRepo/util.py +++ b/MgaRepo/util.py @@ -23,11 +23,11 @@ def commands_getstatusoutput(cmd): text = b'' pipe.stdin.close() while True: - text += os.read(of,8192) + text += os.read(of, 8192) status = pipe.poll() if status is not None or text == '': break - if text[-1:] == '\n': text = text[:-1] + if text[-1:] == b'\n': text = text[:-1] return status, text.decode('utf8') def execcmd(*cmd, **kwargs): @@ -54,10 +54,14 @@ def execcmd(*cmd, **kwargs): r,w,x = select.select((of,ef), (), ()) odata = None if of in r: + # TODO: wait until all text is received before decoding to + # reduce the chance of decoding a partial UTF-8 codepoint odata = (os.read(of, 8192)).decode('utf8') sys.stdout.write(odata) edata = None if ef in r: + # TODO: wait until all text is received before decoding to + # reduce the chance of decoding a partial UTF-8 codepoint edata = (os.read(ef, 8192)).decode('utf8') err.write(edata) sys.stderr.write(edata) @@ -92,10 +96,14 @@ def get_output_exec(cmdstr): r,w,x = select.select((of,ef), (), ()) odata = None if of in r: + # TODO: wait until all text is received before decoding to + # reduce the chance of decoding a partial UTF-8 codepoint odata = (os.read(of, 8192)).decode('utf8') output.write(odata) edata = None if ef in r: + # TODO: wait until all text is received before decoding to + # reduce the chance of decoding a partial UTF-8 codepoint edata = (os.read(ef, 8192)).decode('utf8') err.write(edata) status = p.poll() @@ -135,12 +143,12 @@ def mapurl(url): try: expr_, replace = urlmap.split()[:2] except ValueError: - sys.stderr.buffer.write("invalid url-map: %s" % urlmap) + sys.stderr.buffer.write(("invalid url-map: %s" % urlmap).encode()) else: try: newurl = re.sub(expr_, replace, url) except re.error as errmsg: - sys.stderr.buffer.write("error in URL mapping regexp: %s" % errmsg) + sys.stderr.buffer.write(("error in URL mapping regexp: %s" % errmsg).encode()) return newurl |