diff options
Diffstat (limited to 'MgaRepo')
39 files changed, 4916 insertions, 0 deletions
diff --git a/MgaRepo/ConfigParser.py b/MgaRepo/ConfigParser.py new file mode 100644 index 0000000..bac6d36 --- /dev/null +++ b/MgaRepo/ConfigParser.py @@ -0,0 +1,434 @@ +""" +This is a heavily hacked version of ConfigParser to keep the order in +which options and sections are read, and allow multiple options with +the same key. +""" +from __future__ import generators +import string, types +import re + +__all__ = ["NoSectionError","DuplicateSectionError","NoOptionError", + "InterpolationError","InterpolationDepthError","ParsingError", + "MissingSectionHeaderError","ConfigParser", + "MAX_INTERPOLATION_DEPTH"] + +DEFAULTSECT = "DEFAULT" + +MAX_INTERPOLATION_DEPTH = 10 + +# exception classes +class Error(Exception): + def __init__(self, msg=''): + self._msg = msg + Exception.__init__(self, msg) + def __repr__(self): + return self._msg + __str__ = __repr__ + +class NoSectionError(Error): + def __init__(self, section): + Error.__init__(self, 'No section: %s' % section) + self.section = section + +class DuplicateSectionError(Error): + def __init__(self, section): + Error.__init__(self, "Section %s already exists" % section) + self.section = section + +class NoOptionError(Error): + def __init__(self, option, section): + Error.__init__(self, "No option `%s' in section: %s" % + (option, section)) + self.option = option + self.section = section + +class InterpolationError(Error): + def __init__(self, reference, option, section, rawval): + Error.__init__(self, + "Bad value substitution:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\tkey : %s\n" + "\trawval : %s\n" + % (section, option, reference, rawval)) + self.reference = reference + self.option = option + self.section = section + +class InterpolationDepthError(Error): + def __init__(self, option, section, rawval): + Error.__init__(self, + "Value interpolation too deeply recursive:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\trawval : %s\n" + % (section, option, rawval)) + self.option = option + self.section = section + +class ParsingError(Error): + def __init__(self, filename): + Error.__init__(self, 'File contains parsing errors: %s' % filename) + self.filename = filename + self.errors = [] + + def append(self, lineno, line): + self.errors.append((lineno, line)) + self._msg = self._msg + '\n\t[line %2d]: %s' % (lineno, line) + +class MissingSectionHeaderError(ParsingError): + def __init__(self, filename, lineno, line): + Error.__init__( + self, + 'File contains no section headers.\nfile: %s, line: %d\n%s' % + (filename, lineno, line)) + self.filename = filename + self.lineno = lineno + self.line = line + +class ConfigParser: + def __init__(self, defaults=None): + # Options are stored in __sections_list like this: + # [(sectname, [(optname, optval), ...]), ...] + self.__sections_list = [] + self.__sections_dict = {} + if defaults is None: + self.__defaults = {} + else: + self.__defaults = defaults + + def defaults(self): + return self.__defaults + + def sections(self): + return self.__sections_dict.keys() + + def has_section(self, section): + return self.__sections_dict.has_key(section) + + def options(self, section): + self.__sections_dict[section] + try: + opts = self.__sections_dict[section].keys() + except KeyError: + raise NoSectionError(section) + return self.__defaults.keys()+opts + + def read(self, filenames): + if type(filenames) in types.StringTypes: + filenames = [filenames] + for filename in filenames: + try: + fp = open(filename) + except IOError: + continue + self.__read(fp, filename) + fp.close() + + def readfp(self, fp, filename=None): + if filename is None: + try: + filename = fp.name + except AttributeError: + filename = '<???>' + self.__read(fp, filename) + + def set(self, section, option, value): + if self.__sections_dict.has_key(section): + sectdict = self.__sections_dict[section] + sectlist = [] + self.__sections_list.append((section, sectlist)) + elif section == DEFAULTSECT: + sectdict = self.__defaults + sectlist = None + else: + sectdict = {} + self.__sections_dict[section] = sectdict + sectlist = [] + self.__sections_list.append((section, sectlist)) + xform = self.optionxform(option) + sectdict[xform] = value + if sectlist is not None: + sectlist.append([xform, value]) + + def get(self, section, option, raw=0, vars=None): + d = self.__defaults.copy() + try: + d.update(self.__sections_dict[section]) + except KeyError: + if section != DEFAULTSECT: + raise NoSectionError(section) + if vars: + d.update(vars) + option = self.optionxform(option) + try: + rawval = d[option] + except KeyError: + raise NoOptionError(option, section) + if raw: + return rawval + return self.__interpolate(rawval, d) + + def getall(self, section, option, raw=0, vars=None): + option = self.optionxform(option) + values = [] + d = self.__defaults.copy() + if section != DEFAULTSECT: + for sectname, options in self.__sections_list: + if sectname == section: + for optname, value in options: + if optname == option: + values.append(value) + d[optname] = value + if raw: + return values + if vars: + d.update(vars) + for i in len(values): + values[i] = self.__interpolate(values[i], d) + return values + + def walk(self, section, option=None, raw=0, vars=None): + # Build dictionary for interpolation + try: + d = self.__sections_dict[section].copy() + except KeyError: + if section == DEFAULTSECT: + d = {} + else: + raise NoSectionError(section) + d.update(self.__defaults) + if vars: + d.update(vars) + + # Start walking + if option: + option = self.optionxform(option) + if section != DEFAULTSECT: + for sectname, options in self.__sections_list: + if sectname == section: + for optname, value in options: + if not option or optname == option: + if not raw: + value = self.__interpolate(value, d) + yield (optname, value) + + def __interpolate(self, value, vars): + rawval = value + depth = 0 + while depth < 10: + depth = depth + 1 + if value.find("%(") >= 0: + try: + value = value % vars + except KeyError, key: + raise InterpolationError(key, option, section, rawval) + else: + break + if value.find("%(") >= 0: + raise InterpolationDepthError(option, section, rawval) + return value + + def __get(self, section, conv, option): + return conv(self.get(section, option)) + + def getint(self, section, option): + return self.__get(section, string.atoi, option) + + def getfloat(self, section, option): + return self.__get(section, string.atof, option) + + def getboolean(self, section, option): + states = {'1': 1, 'yes': 1, 'true': 1, 'on': 1, + '0': 0, 'no': 0, 'false': 0, 'off': 0} + v = self.get(section, option) + if not states.has_key(v.lower()): + raise ValueError, 'Not a boolean: %s' % v + return states[v.lower()] + + def optionxform(self, optionstr): + #return optionstr.lower() + return optionstr + + def has_option(self, section, option): + """Check for the existence of a given option in a given section.""" + if not section or section == "DEFAULT": + return self.__defaults.has_key(option) + elif not self.has_section(section): + return 0 + else: + option = self.optionxform(option) + return self.__sections_dict[section].has_key(option) + + SECTCRE = re.compile(r'\[(?P<header>[^]]+)\]') + OPTCRE = re.compile(r'(?P<option>\S+)\s*(?P<vi>[:=])\s*(?P<value>.*)$') + + def __read(self, fp, fpname): + cursectdict = None # None, or a dictionary + optname = None + lineno = 0 + e = None # None, or an exception + while 1: + line = fp.readline() + if not line: + break + lineno = lineno + 1 + # comment or blank line? + if line.strip() == '' or line[0] in '#;': + continue + if line.split()[0].lower() == 'rem' \ + and line[0] in "rR": # no leading whitespace + continue + # continuation line? + if line[0] in ' \t' and cursectdict is not None and optname: + value = line.strip() + if value: + k = self.optionxform(optname) + cursectdict[k] = "%s\n%s" % (cursectdict[k], value) + cursectlist[-1][1] = "%s\n%s" % (cursectlist[-1][1], value) + # a section header or option header? + else: + # is it a section header? + mo = self.SECTCRE.match(line) + if mo: + sectname = mo.group('header') + if self.__sections_dict.has_key(sectname): + cursectdict = self.__sections_dict[sectname] + cursectlist = [] + self.__sections_list.append((sectname, cursectlist)) + elif sectname == DEFAULTSECT: + cursectdict = self.__defaults + cursectlist = None + else: + cursectdict = {} + self.__sections_dict[sectname] = cursectdict + cursectlist = [] + self.__sections_list.append((sectname, cursectlist)) + # So sections can't start with a continuation line + optname = None + # no section header in the file? + elif cursectdict is None: + raise MissingSectionHeaderError(fpname, lineno, `line`) + # an option line? + else: + mo = self.OPTCRE.match(line) + if mo: + optname, vi, optval = mo.group('option', 'vi', 'value') + if vi in ('=', ':') and ';' in optval: + # ';' is a comment delimiter only if it follows + # a spacing character + pos = optval.find(';') + if pos and optval[pos-1] in string.whitespace: + optval = optval[:pos] + optval = optval.strip() + # allow empty values + if optval == '""': + optval = '' + xform = self.optionxform(optname) + cursectdict[xform] = optval + if cursectlist is not None: + cursectlist.append([xform, optval]) + else: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + if not e: + e = ParsingError(fpname) + e.append(lineno, `line`) + # if any parsing errors occurred, raise an exception + if e: + raise e + +# Here we wrap this hacked ConfigParser into something more useful +# for us. + +import os + +class Config: + def __init__(self): + self._config = ConfigParser() + self._wrapped = {} + conffiles = [] + mgarepo_conf = os.environ.get("MGAREPO_CONF") + if mgarepo_conf: + conffiles.append(mgarepo_conf) + else: + conffiles.append("/etc/mgarepo.conf") + conffiles.append(os.path.expanduser("~/.mgarepo/config")) + for file in conffiles: + if os.path.isfile(file): + self._config.read(file) + + def wrap(self, section, handler, option=None): + """Set one wrapper for a given section + + The wrapper must be a function + f(section, option=None, default=None, walk=False). + """ + self._wrapped[section] = handler + + def sections(self): + try: + return self._config.sections() + except Error: + return [] + + def options(self, section): + try: + return self._config.options(section) + except Error: + return [] + + def set(self, section, option, value): + return self._config.set(section, option, value) + + def walk(self, section, option=None, raw=0, vars=None): + handler = self._wrapped.get(section) + if handler: + return handler(section, option, walk=True) + return self._config.walk(section, option, raw, vars) + + def get(self, section, option, default=None, raw=False, wrap=True): + if wrap: + handler = self._wrapped.get(section) + if handler: + handler = self._wrapped.get(section) + return handler(section, option, default) + try: + return self._config.get(section, option, raw=raw) + except Error: + return default + + def getint(self, section, option, default=None): + ret = self.get(section, option, default) + if type(ret) == type(""): + return int(ret) + + def getbool(self, section, option, default=None): + ret = self.get(section, option, default) + states = {'1': 1, 'yes': 1, 'true': 1, 'on': 1, + '0': 0, 'no': 0, 'false': 0, 'off': 0} + if type(ret) == type("") and states.has_key(ret.lower()): + return states[ret.lower()] + return default + +def test(): + config = Config() + def handler(section, option=None, default=None, walk=False): + d = {"fulano": "ciclano", + "foolano": "ceeclano"} + if walk: + return d.items() + elif option in d: + return d[option] + else: + return config.get(section, option, default, wrap=False) + config.wrap("users", handler=handler) + print config.get("users", "fulano") # found in wrapper + print config.get("users", "andreas") # found in repsys.conf + print config.walk("users") + +if __name__ == "__main__": + test() +# vim:ts=4:sw=4:et diff --git a/MgaRepo/__init__.py b/MgaRepo/__init__.py new file mode 100644 index 0000000..b0df184 --- /dev/null +++ b/MgaRepo/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +import re +import os +import tempfile + +import ConfigParser + +config = ConfigParser.Config() +tempfile.tempdir = config.get("global", "tempdir", None) or None # when "" +del ConfigParser + +def disable_mirror(*a, **kw): + config.set("global", "use-mirror", "no") + +class Error(Exception): pass + +class SilentError(Error): pass + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/binrepo.py b/MgaRepo/binrepo.py new file mode 100644 index 0000000..9186980 --- /dev/null +++ b/MgaRepo/binrepo.py @@ -0,0 +1,395 @@ +from MgaRepo import Error, config, mirror, layout +from MgaRepo.util import execcmd, rellink +from MgaRepo.svn import SVN + +import sys +import os +import string +import stat +import shutil +import re +import tempfile +import hashlib +import urlparse +import threading +from cStringIO import StringIO + +DEFAULT_TARBALLS_REPO = "/tarballs" +BINARIES_DIR_NAME = "SOURCES" +BINARIES_CHECKOUT_NAME = "SOURCES-bin" + +PROP_USES_BINREPO = "mdv:uses-binrepo" +PROP_BINREPO_REV = "mdv:binrepo-rev" + +BINREPOS_SECTION = "binrepos" + +SOURCES_FILE = "sha1.lst" + +class ChecksumError(Error): + pass + +def svn_baseurl(target): + svn = SVN() + info = svn.info2(target) + if info is None: + # unversioned resource + newtarget = os.path.dirname(target) + info = svn.info2(newtarget) + assert info is not None, "svn_basedir should not be used with a "\ + "non-versioned directory" + root = info["Repository Root"] + url = info["URL"] + kind = info["Node Kind"] + path = url[len(root):] + if kind == "directory": + return url + basepath = os.path.dirname(path) + baseurl = mirror.normalize_path(url + "/" + basepath) + return baseurl + +def svn_root(target): + svn = SVN() + info = svn.info2(target) + if info is None: + newtarget = os.path.dirname(target) + info = svn.info2(newtarget) + assert info is not None + return info["Repository Root"] + +def enabled(url): + #TODO use information from url to find out whether we have a binrepo + # available for this url + use = config.getbool("global", "use-binaries-repository", False) + return use + +def default_repo(): + base = config.get("global", "binaries-repository", None) + if base is None: + default_parent = config.get("global", "default_parent", None) + if default_parent is None: + raise Error, "no binaries-repository nor default_parent "\ + "configured" + comps = urlparse.urlparse(default_parent) + base = comps[1] + ":" + DEFAULT_TARBALLS_REPO + return base + +def translate_url(url): + url = mirror.normalize_path(url) + main = mirror.normalize_path(layout.repository_url()) + subpath = url[len(main)+1:] + # [binrepos] + # updates/2009.0 = svn+ssh://svn.mandriva.com/svn/binrepo/20090/ + ## svn+ssh://svn.mandriva.com/svn/packages/2009.0/trafshow/current + ## would translate to + ## svn+ssh://svn.mandriva.com/svn/binrepo/20090/updates/trafshow/current/ + binbase = None + if BINREPOS_SECTION in config.sections(): + for option, value in config.walk(BINREPOS_SECTION): + if subpath.startswith(option): + binbase = value + break + binurl = mirror._joinurl(binbase or default_repo(), subpath) + return binurl + +def translate_topdir(path): + """Returns the URL in the binrepo from a given path inside a SVN + checkout directory. + + @path: if specified, returns a URL in the binrepo whose path is the + same as the path inside the main repository. + """ + baseurl = svn_baseurl(path) + binurl = translate_url(baseurl) + target = mirror.normalize_path(binurl) + return target + +def is_binary(path): + raw = config.get("binrepo", "upload-match", + "\.(7z|Z|bin|bz2|cpio|db|deb|egg|gem|gz|jar|jisp|lzma|"\ + "pdf|pgn\\.gz|pk3|png|rpm|run|sdz|smzip|tar|tbz|"\ + "tbz2|tgz|ttf|uqm|wad|war|xar|xpi|xz|zip|wav|mp3|ogg|"\ + "jpg|png|gif|avi|mpg|mpeg|rar)$") + maxsize = config.getint("binrepo", "upload-match-size", "1048576") # 1MiB + expr = re.compile(raw) + name = os.path.basename(path) + if expr.search(name): + return True + st = os.stat(path) + if st[stat.ST_SIZE] >= maxsize: + return True + return False + +def find_binaries(paths): + new = [] + for path in paths: + if os.path.isdir(path): + for name in os.listdir(path): + fpath = os.path.join(path, name) + if is_binary(fpath): + new.append(fpath) + else: + if is_binary(path): + new.append(path) + return new + +def make_symlinks(source, dest): + todo = [] + tomove = [] + for name in os.listdir(source): + path = os.path.join(source, name) + if not os.path.isdir(path) and not name.startswith("."): + destpath = os.path.join(dest, name) + linkpath = rellink(path, destpath) + if os.path.exists(destpath): + if (os.path.islink(destpath) and + os.readlink(destpath) == linkpath): + continue + movepath = destpath + ".repsys-moved" + if os.path.exists(movepath): + raise Error, "cannot create symlink, %s already "\ + "exists (%s too)" % (destpath, movepath) + tomove.append((destpath, movepath)) + todo.append((destpath, linkpath)) + for destpath, movepath in tomove: + os.rename(destpath, movepath) + for destpath, linkpath in todo: + os.symlink(linkpath, destpath) + +def download(targetdir, pkgdirurl=None, export=False, show=True, + revision=None, symlinks=True, check=False): + assert not export or (export and pkgdirurl) + svn = SVN() + sourcespath = os.path.join(targetdir, "SOURCES") + binpath = os.path.join(targetdir, BINARIES_CHECKOUT_NAME) + if pkgdirurl: + topurl = translate_url(pkgdirurl) + else: + topurl = translate_topdir(targetdir) + binrev = None + if revision: + if pkgdirurl: + binrev = mapped_revision(pkgdirurl, revision) + else: + binrev = mapped_revision(targetdir, revision, wc=True) + binurl = mirror._joinurl(topurl, BINARIES_DIR_NAME) + if export: + svn.export(binurl, binpath, rev=binrev, show=show) + else: + svn.checkout(binurl, binpath, rev=binrev, show=show) + if symlinks: + make_symlinks(binpath, sourcespath) + if check: + check_sources(targetdir) + +def import_binaries(topdir, pkgname): + """Import all binaries from a given package checkout + + (with pending svn adds) + + @topdir: the path to the svn checkout + """ + svn = SVN() + topurl = translate_topdir(topdir) + sourcesdir = os.path.join(topdir, "SOURCES") + bintopdir = tempfile.mktemp("repsys") + try: + svn.checkout(topurl, bintopdir) + checkout = True + except Error: + bintopdir = tempfile.mkdtemp("repsys") + checkout = False + try: + bindir = os.path.join(bintopdir, BINARIES_DIR_NAME) + if not os.path.exists(bindir): + if checkout: + svn.mkdir(bindir) + else: + os.mkdir(bindir) + binaries = find_binaries([sourcesdir]) + update = update_sources_threaded(topdir, added=binaries) + for path in binaries: + name = os.path.basename(path) + binpath = os.path.join(bindir, name) + os.rename(path, binpath) + try: + svn.remove(path) + except Error: + # file not tracked + svn.revert(path) + if checkout: + svn.add(binpath) + log = "imported binaries for %s" % pkgname + if checkout: + rev = svn.commit(bindir, log=log) + else: + rev = svn.import_(bintopdir, topurl, log=log) + svn.propset(PROP_USES_BINREPO, "yes", topdir) + svn.propset(PROP_BINREPO_REV, str(rev), topdir) + update.join() + svn.add(sources_path(topdir)) + finally: + shutil.rmtree(bintopdir) + +def create_package_dirs(bintopdir): + svn = SVN() + binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME) + silent = config.get("log", "ignore-string", "SILENT") + message = "%s: created binrepo package structure" % silent + svn.mkdir(binurl, log=message, parents=True) + +def parse_sources(path): + entries = {} + f = open(path) + for rawline in f: + line = rawline.strip() + try: + sum, name = line.split(None, 1) + except ValueError: + # failed to unpack, line format error + raise Error, "invalid line in sources file: %s" % rawline + entries[name] = sum + return entries + +def check_hash(path, sum): + newsum = file_hash(path) + if newsum != sum: + raise ChecksumError, "different checksums for %s: expected %s, "\ + "but %s was found" % (path, sum, newsum) + +def check_sources(topdir): + spath = sources_path(topdir) + if not os.path.exists(spath): + raise Error, "'%s' was not found" % spath + entries = parse_sources(spath) + for name, sum in entries.iteritems(): + fpath = os.path.join(topdir, "SOURCES", name) + check_hash(fpath, sum) + +def file_hash(path): + sum = hashlib.sha1() + f = open(path) + while True: + block = f.read(4096) + if not block: + break + sum.update(block) + f.close() + return sum.hexdigest() + +def sources_path(topdir): + path = os.path.join(topdir, "SOURCES", SOURCES_FILE) + return path + +def update_sources(topdir, added=[], removed=[]): + path = sources_path(topdir) + entries = {} + if os.path.isfile(path): + entries = parse_sources(path) + f = open(path, "w") # open before calculating hashes + for name in removed: + entries.pop(removed) + for added_path in added: + name = os.path.basename(added_path) + entries[name] = file_hash(added_path) + for name in sorted(entries): + f.write("%s %s\n" % (entries[name], name)) + f.close() + +def update_sources_threaded(*args, **kwargs): + t = threading.Thread(target=update_sources, args=args, kwargs=kwargs) + t.start() + t.join() + return t + +def upload(path, message=None): + from MgaRepo.rpmutil import getpkgtopdir + svn = SVN() + if not os.path.exists(path): + raise Error, "not found: %s" % path + # XXX check if the path is under SOURCES/ + paths = find_binaries([path]) + if not paths: + raise Error, "'%s' does not seem to have any tarballs" % path + topdir = getpkgtopdir() + bintopdir = translate_topdir(topdir) + binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME) + sourcesdir = os.path.join(topdir, "SOURCES") + bindir = os.path.join(topdir, BINARIES_CHECKOUT_NAME) + silent = config.get("log", "ignore-string", "SILENT") + if not os.path.exists(bindir): + try: + download(topdir, show=False) + except Error: + # possibly the package does not exist + # (TODO check whether it is really a 'path not found' error) + pass + if not os.path.exists(bindir): + create_package_dirs(bintopdir) + svn.propset(PROP_USES_BINREPO, "yes", topdir) + svn.commit(topdir, log="%s: created binrepo structure" % silent) + download(topdir, show=False) + for path in paths: + if svn.info2(path): + sys.stderr.write("'%s' is already tracked by svn, ignoring\n" % + path) + continue + name = os.path.basename(path) + binpath = os.path.join(bindir, name) + os.rename(path, binpath) + svn.add(binpath) + if not message: + message = "%s: new binary files %s" % (silent, " ".join(paths)) + make_symlinks(bindir, sourcesdir) + update = update_sources_threaded(topdir, added=paths) + rev = svn.commit(binpath, log=message) + svn.propset(PROP_BINREPO_REV, str(rev), topdir) + sources = sources_path(topdir) + svn.add(sources) + update.join() + svn.commit(topdir + " " + sources, log=message, nonrecursive=True) + +def mapped_revision(target, revision, wc=False): + """Maps a txtrepo revision to a binrepo datespec + + This datespec can is intended to be used by svn .. -r DATE. + + @target: a working copy path or a URL + @revision: if target is a URL, the revision number used when fetching + svn info + @wc: if True indicates that 'target' must be interpreted as a + the path of a svn working copy, otherwise it is handled as a URL + """ + svn = SVN() + binrev = None + if wc: + spath = sources_path(target) + if os.path.exists(spath): + infolines = svn.info(spath, xml=True) + if infolines: + rawinfo = "".join(infolines) # arg! + found = re.search("<date>(.*?)</date>", rawinfo).groups() + date = found[0] + else: + raise Error, "bogus 'svn info' for '%s'" % spath + else: + raise Error, "'%s' was not found" % spath + else: + url = mirror._joinurl(target, sources_path("")) + date = svn.propget("svn:date", url, rev=revision, revprop=True) + if not date: + raise Error, "no valid date available for '%s'" % url + binrev = "{%s}" % date + return binrev + +def markrelease(sourceurl, releasesurl, version, release, revision): + svn = SVN() + binrev = mapped_revision(sourceurl, revision) + binsource = translate_url(sourceurl) + binreleases = translate_url(releasesurl) + versiondir = mirror._joinurl(binreleases, version) + dest = mirror._joinurl(versiondir, release) + svn.mkdir(binreleases, noerror=1, log="created directory for releases") + svn.mkdir(versiondir, noerror=1, log="created directory for version %s" % version) + svn.copy(binsource, dest, rev=binrev, + log="%%markrelease ver=%s rel=%s rev=%s binrev=%s" % (version, release, + revision, binrev)) diff --git a/MgaRepo/cgi/__init__.py b/MgaRepo/cgi/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/MgaRepo/cgi/__init__.py diff --git a/MgaRepo/cgi/soapserver.py b/MgaRepo/cgi/soapserver.py new file mode 100644 index 0000000..a203940 --- /dev/null +++ b/MgaRepo/cgi/soapserver.py @@ -0,0 +1,93 @@ +#!/usr/bin/python +from MgaRepo import Error, config +from MgaRepo.rpmutil import get_srpm +from MgaRepo.cgiutil import CgiError, get_targets +import sys +import os + +try: + import NINZ.dispatch +except ImportError: + NINZ = None + +class SoapIface: + def author_email(self, author): + return config.get("users", author) + + def submit_package(self, packageurl, packagerev, targetname): + username = os.environ.get("REMOTE_USER") + packager = config.get("users", username) + if not packager: + raise CgiError, "your email was not found" + elif not packagerev: + raise CgiError, "no revision provided" + elif not targetname: + raise CgiError, "no target provided" + else: + targetname = targetname.lower() + for target in get_targets(): + if target.name.lower() == targetname: + break + else: + raise CgiError, "target not found" + try: + tmp = int(packagerev) + except ValueError: + raise CgiError, "invalid revision provided" + for allowed in target.allowed: + if packageurl.startswith(allowed): + break + else: + raise CgiError, "%s is not allowed for this target" \ + % packageurl + get_srpm(packageurl, + revision=packagerev, + targetdirs=target.target, + packager=packager, + revname=1, + svnlog=1, + scripts=target.scripts) + return 1 + + def submit_targets(self): + return [x.name for x in get_targets()] + +TEMPLATE = """\ +Content-type: text/html + +<html> +<head> +<title>Repository system SOAP server</title> +</head> +<body bgcolor="white"> +<br> +<hr> +<center> +<b>%(message)s</b> +</center> +<hr> +</body> +</html> +""" + +def show(msg="", error=0): + if error: + msg = '<font color="red">%s</font>' % msg + print TEMPLATE % {"message":msg} + +def main(): + if not os.environ.has_key('REQUEST_METHOD'): + sys.stderr.write("error: this program is meant to be used as a cgi\n") + sys.exit(1) + if not NINZ: + show("NINZ is not properly installed in this system", error=1) + sys.exit(1) + username = os.environ.get("REMOTE_USER") + method = os.environ.get("REQUEST_METHOD") + if not username or method != "POST": + show("This is a SOAP interface!", error=1) + sys.exit(1) + + NINZ.dispatch.AsCGI(modules=(SoapIface(),)) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/cgi/submit.py b/MgaRepo/cgi/submit.py new file mode 100644 index 0000000..cba5977 --- /dev/null +++ b/MgaRepo/cgi/submit.py @@ -0,0 +1,119 @@ +#!/usr/bin/python +from MgaRepo import Error, config +from MgaRepo.rpmutil import get_srpm +from MgaRepo.cgiutil import CgiError, get_targets +import cgi +import sys +import os + +TEMPLATE = """\ +<html> +<head> +<title>Repository package submission system</title> +</head> +<body bgcolor="white"> +<table cellspacing=0 cellpadding=0 border=0 width="100%%"> + <tr bgcolor="#020264"><td align="left" valign="middle"><img src="http://qa.mandriva.com/mandriva.png" hspace=0 border=0 alt=""></td></tr> +</table> +<br> +<hr> +<center> +<b>%(message)s</b> +<br><br> +<form method="POST" action=""> +<table><tr><td valign="top"> + Package URL:<br> + <input name="packageurl" size="60" value="svn+ssh://cvs.mandriva.com/svn/mdv/cooker/"><br> + <small>Ex. svn+ssh://cvs.mandriva.com/svn/mdv/cooker/pkgname</small><br> + </td><td valign="top"> + Revision:<br> + <input name="packagerev" size="10" value=""><br> + </td></tr></table> + <br> + Package target:<br> + <select name="target" size=5> + %(targetoptions)s + </select><br> + <br> + <input type="submit" value="Submit package"> +</form> +</center> +<hr/> +</body> +</html> +""" + +def get_targetoptions(): + s = "" + selected = " selected" + for target in get_targets(): + s += '<option value="%s"%s>%s</option>' \ + % (target.name, selected, target.name) + selected = "" + return s + +def show(msg="", error=0): + if error: + msg = '<font color="red">%s</font>' % msg + print TEMPLATE % {"message":msg, "targetoptions":get_targetoptions()} + +def submit_packages(packager): + form = cgi.FieldStorage() + packageurl = form.getfirst("packageurl", "").strip() + packagerev = form.getfirst("packagerev", "").strip() + if not packageurl: + show() + elif not packagerev: + raise CgiError, "No revision provided!" + else: + targetname = form.getfirst("target") + if not targetname: + raise CgiError, "No target selected!" + for target in get_targets(): + if target.name == targetname: + break + else: + raise CgiError, "Target not found!" + try: + tmp = int(packagerev) + except ValueError: + raise CgiError, "Invalid revision provided!" + for allowed in target.allowed: + if packageurl.startswith(allowed): + break + else: + raise CgiError, "%s is not allowed for this target!" % packageurl + get_srpm(packageurl, + revision=packagerev, + targetdirs=target.target, + packager=packager, + revname=1, + svnlog=1, + scripts=target.scripts) + show("Package submitted!") + +def main(): + if not os.environ.has_key('REQUEST_METHOD'): + sys.stderr.write("error: this program is meant to be used as a cgi\n") + sys.exit(1) + print "Content-type: text/html\n\n" + try: + username = os.environ.get("REMOTE_USER") + method = os.environ.get("REQUEST_METHOD") + if not username or method != "POST": + show() + else: + useremail = config.get("users", username) + if not useremail: + raise CgiError, \ + "Your email was not found. Contact the administrator!" + submit_packages(useremail) + except CgiError, e: + show(str(e), error=1) + except Error, e: + error = str(e) + show(error[0].upper()+error[1:], error=1) + except: + cgi.print_exception() + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/cgi/xmlrpcserver.py b/MgaRepo/cgi/xmlrpcserver.py new file mode 100644 index 0000000..8521b4f --- /dev/null +++ b/MgaRepo/cgi/xmlrpcserver.py @@ -0,0 +1,111 @@ +#!/usr/bin/python +from MgaRepo import Error, config +from MgaRepo.rpmutil import get_srpm +from MgaRepo.cgiutil import CgiError, get_targets +import sys +import os + +import xmlrpclib, cgi + +class XmlRpcIface: + def author_email(self, author): + return config.get("users", author) + + def submit_package(self, packageurl, packagerev, targetname): + username = os.environ.get("REMOTE_USER") + packager = config.get("users", username) + if not packager: + raise CgiError, "your email was not found" + elif not packagerev: + raise CgiError, "no revision provided" + elif not targetname: + raise CgiError, "no target provided" + else: + targetname = targetname.lower() + for target in get_targets(): + if target.name.lower() == targetname: + break + else: + raise CgiError, "target not found" + try: + tmp = int(packagerev) + except ValueError: + raise CgiError, "invalid revision provided" + for allowed in target.allowed: + if packageurl.startswith(allowed): + break + else: + raise CgiError, "%s is not allowed for this target" \ + % packageurl + get_srpm(packageurl, + revision=packagerev, + targetdirs=target.target, + packager=packager, + revname=1, + svnlog=1, + scripts=target.scripts) + return 1 + + def submit_targets(self): + return [x.name for x in get_targets()] + +TEMPLATE = """\ +Content-type: text/html + +<html> +<head> +<title>Repository system SOAP server</title> +</head> +<body bgcolor="white"> +<br> +<hr> +<center> +<b>%(message)s</b> +</center> +<hr> +</body> +</html> +""" + +def show(msg="", error=0): + if error: + msg = '<font color="red">%s</font>' % msg + print TEMPLATE % {"message":msg} + +def main(): + if not os.environ.has_key('REQUEST_METHOD'): + sys.stderr.write("error: this program is meant to be used as a cgi\n") + sys.exit(1) + username = os.environ.get("REMOTE_USER") + method = os.environ.get("REQUEST_METHOD") + if not username or method != "POST": + show("This is a XMLRPC interface!", error=1) + sys.exit(1) + + iface = XmlRpcIface() + + response = "" + try: + form = cgi.FieldStorage() + parms, method = xmlrpclib.loads(form.value) + meth = getattr(iface, method) + response = (meth(*parms),) + except CgiError, e: + msg = str(e) + try: + msg = msg.decode("iso-8859-1") + except UnicodeError: + pass + response = xmlrpclib.Fault(1, msg) + except Exception, e: + msg = str(e) + try: + msg = msg.decode("iso-8859-1") + except UnicodeError: + pass + response = xmlrpclib.Fault(1, msg) + + sys.stdout.write("Content-type: text/xml\n\n") + sys.stdout.write(xmlrpclib.dumps(response, methodresponse=1)) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/cgiutil.py b/MgaRepo/cgiutil.py new file mode 100644 index 0000000..68afdcd --- /dev/null +++ b/MgaRepo/cgiutil.py @@ -0,0 +1,53 @@ +#!/usr/bin/python +from MgaRepo import Error, config +from MgaRepo.svn import SVN +from MgaRepo.ConfigParser import NoSectionError +import time +import re + +class CgiError(Error): pass + +class SubmitTarget: + def __init__(self): + self.name = "" + self.target = "" + self.macros = [] + self.allowed = [] + self.scripts = [] + +TARGETS = [] + +def parse_macrosref(refs, config): + macros = [] + for name in refs: + secname = "macros %s" % name + try: + macros.extend(config.walk(secname, raw=True)) + except NoSectionError: + raise Error, "missing macros section " \ + "%r in configuration" % secname + return macros + +def get_targets(): + global TARGETS + if not TARGETS: + target = SubmitTarget() + targetoptions = {} + submit_re = re.compile("^submit\s+(.+)$") + for section in config.sections(): + m = submit_re.match(section) + if m: + target = SubmitTarget() + target.name = m.group(1) + for option, value in config.walk(section): + if option in ("target", "allowed", "scripts"): + setattr(target, option, value.split()) + elif option == "rpm-macros": + refs = value.split() + target.macros = parse_macrosref(refs, config) + else: + raise Error, "unknown [%s] option %s" % (section, option) + TARGETS.append(target) + return TARGETS + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/command.py b/MgaRepo/command.py new file mode 100644 index 0000000..9028b3b --- /dev/null +++ b/MgaRepo/command.py @@ -0,0 +1,61 @@ +#!/usr/bin/python +from MgaRepo import SilentError, Error, config +import sys, os +import urlparse +import optparse + +__all__ = ["OptionParser", "do_command", "default_parent"] + +class CapitalizeHelpFormatter(optparse.IndentedHelpFormatter): + + def format_usage(self, usage): + return optparse.IndentedHelpFormatter \ + .format_usage(self, usage).capitalize() + + def format_heading(self, heading): + return optparse.IndentedHelpFormatter \ + .format_heading(self, heading).capitalize() + +class OptionParser(optparse.OptionParser): + + def __init__(self, usage=None, help=None, **kwargs): + if not "formatter" in kwargs: + kwargs["formatter"] = CapitalizeHelpFormatter() + optparse.OptionParser.__init__(self, usage, **kwargs) + self._overload_help = help + + def format_help(self, formatter=None): + if self._overload_help: + return self._overload_help + else: + return optparse.OptionParser.format_help(self, formatter) + + def error(self, msg): + raise Error, msg + +def do_command(parse_options_func, main_func): + try: + opt = parse_options_func() + main_func(**opt.__dict__) + except SilentError: + sys.exit(1) + except Error, e: + sys.stderr.write("error: %s\n" % str(e)) + sys.exit(1) + except KeyboardInterrupt: + sys.stderr.write("interrupted\n") + sys.stderr.flush() + sys.exit(1) + +def default_parent(url): + if url.find("://") == -1: + default_parent = config.get("global", "default_parent") + if not default_parent: + raise Error, "received a relative url, " \ + "but default_parent was not setup" + parsed = list(urlparse.urlparse(default_parent)) + parsed[2] = os.path.normpath(parsed[2] + "/" + url) + url = urlparse.urlunparse(parsed) + return url + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/__init__.py b/MgaRepo/commands/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/MgaRepo/commands/__init__.py diff --git a/MgaRepo/commands/authoremail.py b/MgaRepo/commands/authoremail.py new file mode 100644 index 0000000..edfe332 --- /dev/null +++ b/MgaRepo/commands/authoremail.py @@ -0,0 +1,37 @@ +#!/usr/bin/python +from MgaRepo import Error, config +from MgaRepo.command import * +import sys +import getopt + +HELP = """\ +Usage: mgarepo authoremail [OPTIONS] AUTHOR + +Shows the e-mail of an SVN author. It is just a simple interface to access +the [authors] section of mgarepo.conf. + +Options: + -h Show this message + +Examples: + mgarepo authoremail john +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + if len(args) != 1: + raise Error, "invalid arguments" + opts.author = args[0] + return opts + +def print_author_email(author): + email = config.get("users", author) + if not email: + raise Error, "author not found" + print email + +def main(): + do_command(parse_options, print_author_email) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/changed.py b/MgaRepo/commands/changed.py new file mode 100644 index 0000000..a17226b --- /dev/null +++ b/MgaRepo/commands/changed.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +from MgaRepo import Error, disable_mirror +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.rpmutil import check_changed +import getopt +import sys + +HELP = """\ +Usage: mgarepo changed [OPTIONS] URL + +Shows if there are pending changes since the last package release. + +Options: + -a Check all packages in given URL + -s Show differences + -M Do not use the mirror (use the main repository) + -h Show this message + +Examples: + mgarepo changed http://repos/svn/cnc/snapshot/foo + mgarepo changed -a http://repos/svn/cnc/snapshot +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-a", dest="all", action="store_true") + parser.add_option("-s", dest="show", action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + opts, args = parser.parse_args() + if len(args) != 1: + raise Error, "invalid arguments" + opts.pkgdirurl = package_url(args[0]) + opts.verbose = 1 # Unconfigurable + return opts + +def main(): + do_command(parse_options, check_changed) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/ci.py b/MgaRepo/commands/ci.py new file mode 100644 index 0000000..e64ac4e --- /dev/null +++ b/MgaRepo/commands/ci.py @@ -0,0 +1,35 @@ +#!/usr/bin/python +from MgaRepo.command import * +from MgaRepo.rpmutil import commit + +HELP = """\ +Usage: mgarepo ci [TARGET] + +Will commit recent modifications in the package. + +The difference between an ordinary "svn ci" and "mgarepo ci" is that it +relocates the working copy to the default repository in case the option +"mirror" is set in mgarepo.conf. + +Options: + -h Show this message + -m MSG Use the MSG as the log message + -F FILE Read log message from FILE + +Examples: + mgarepo ci + mgarepo ci SPECS/package.spec SPECS/package-patch.patch +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-m", dest="message", default=None) + parser.add_option("-F", dest="logfile", type="string", + default=None) + opts, args = parser.parse_args() + if len(args): + opts.target = args[0] + return opts + +def main(): + do_command(parse_options, commit) diff --git a/MgaRepo/commands/co.py b/MgaRepo/commands/co.py new file mode 100644 index 0000000..e7c9aec --- /dev/null +++ b/MgaRepo/commands/co.py @@ -0,0 +1,67 @@ +#!/usr/bin/python +from MgaRepo import Error, disable_mirror +from MgaRepo.command import * +from MgaRepo.rpmutil import checkout +import getopt +import sys + +HELP = """\ +Usage: mgarepo co [OPTIONS] URL [LOCALPATH] + +Checkout the package source from the Mageia repository. + +If the 'mirror' option is enabled, the package is obtained from the mirror +repository. + +You can specify the distro branch to checkout from by using distro/pkgname. + +Options: + -d The distribution branch to checkout from + -b The package branch + -r REV Revision to checkout + -S Do not download sources from the binaries repository + -L Do not make symlinks of the binaries downloaded in SOURCES/ + -s Only checkout the SPECS/ directory + -M Do not use the mirror (use the main repository) + --check Check integrity of files fetched from the binary repository + -h Show this message + +Examples: + mgarepo co pkgname + mgarepo co -d 2009.0 pkgname + mgarepo co 2009.0/pkgame + mgarepo co http://repos/svn/cnc/snapshot/foo + mgarepo co http://repos/svn/cnc/snapshot/foo foo-pkg +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-r", dest="revision") + parser.add_option("-S", dest="use_binrepo", default=True, + action="store_false") + parser.add_option("--check", dest="binrepo_check", default=False, + action="store_true") + parser.add_option("-L", dest="binrepo_link", default=True, + action="store_false") + parser.add_option("--distribution", "-d", dest="distro", default=None) + parser.add_option("--branch", "-b", dest="branch", default=None) + parser.add_option("-s", "--spec", dest="spec", default=False, + action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + opts, args = parser.parse_args() + if len(args) not in (1, 2): + raise Error, "invalid arguments" + # here we don't use package_url in order to notify the user we are + # using the mirror + opts.pkgdirurl = args[0] + if len(args) == 2: + opts.path = args[1] + else: + opts.path = None + return opts + +def main(): + do_command(parse_options, checkout) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/create.py b/MgaRepo/commands/create.py new file mode 100644 index 0000000..da8f16c --- /dev/null +++ b/MgaRepo/commands/create.py @@ -0,0 +1,34 @@ +#!/usr/bin/python +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.rpmutil import create_package +import getopt +import sys + +HELP = """\ +Usage: mgarepo create [OPTIONS] URL + +Creates the minimal structure of a package in the repository. + +Options: + -h Show this message + +Examples: + mgarepo create newpkg + mgarepo create svn+ssh://svn.mageia.org/svn/packages/cauldron/newpkg +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + if len(args) != 1: + raise Error, "invalid arguments" + opts.pkgdirurl = package_url(args[0], mirrored=False) + opts.verbose = 1 # Unconfigurable + return opts + +def main(): + do_command(parse_options, create_package) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/del.py b/MgaRepo/commands/del.py new file mode 100644 index 0000000..79d9d3f --- /dev/null +++ b/MgaRepo/commands/del.py @@ -0,0 +1,30 @@ +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.rpmutil import binrepo_delete + +HELP = """\ +Usage: mgarepo del [OPTIONS] [PATH] + +Remove a given file from the binary sources repository. + +Changes in the sources file will be left uncommited. + +Options: + -c automatically commit the 'sources' file + -h help + +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-c", dest="commit", default=False, + action="store_true") + opts, args = parser.parse_args() + if len(args): + opts.paths = args + else: + raise Error, "you need to provide a path" + return opts + +def main(): + do_command(parse_options, binrepo_delete) diff --git a/MgaRepo/commands/editlog.py b/MgaRepo/commands/editlog.py new file mode 100644 index 0000000..c58c16b --- /dev/null +++ b/MgaRepo/commands/editlog.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.svn import SVN +import re + +HELP = """\ +Usage: mgarepo editlog [OPTIONS] [URL] REVISION + +Options: + -h Show this message + +Examples: + mgarepo editlog 14800 + mgarepo editlog https://repos/svn/cnc/snapshot 14800 +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + if len(args) == 2: + pkgdirurl, revision = args + elif len(args) == 1: + pkgdirurl, revision = "", args[0] + else: + raise Error, "invalid arguments" + opts.pkgdirurl = package_url(pkgdirurl, mirrored=False) + opts.revision = re.compile(r".*?(\d+).*").sub(r"\1", revision) + return opts + +def editlog(pkgdirurl, revision): + svn = SVN() + svn.propedit("svn:log", pkgdirurl, rev=revision) + +def main(): + do_command(parse_options, editlog) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/getspec.py b/MgaRepo/commands/getspec.py new file mode 100644 index 0000000..08407bc --- /dev/null +++ b/MgaRepo/commands/getspec.py @@ -0,0 +1,38 @@ +#!/usr/bin/python +from MgaRepo import Error, disable_mirror +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.rpmutil import get_spec +import getopt +import sys + +HELP = """\ +Usage: mgarepo getspec [OPTIONS] REPPKGURL + +Prints the .spec file of a given package. + +Options: + -t DIR Use DIR as target for spec file (default is ".") + -M Do not use the mirror (use the main repository) + -h Show this message + +Examples: + mgarepo getspec pkgname + mgarepo getspec svn+ssh://svn.mageia.org/svn/packages/cauldron/pkgname +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-t", dest="targetdir", default=".") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + opts, args = parser.parse_args() + if len(args) != 1: + raise Error, "invalid arguments" + opts.pkgdirurl = package_url(args[0]) + return opts + +def main(): + do_command(parse_options, get_spec) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/getsrpm.py b/MgaRepo/commands/getsrpm.py new file mode 100644 index 0000000..6c238ba --- /dev/null +++ b/MgaRepo/commands/getsrpm.py @@ -0,0 +1,100 @@ +#!/usr/bin/python +# +# This program will extract given version/revision of the named package +# from the Conectiva Linux repository system. +# +from MgaRepo import Error, config, disable_mirror +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.rpmutil import get_srpm +import tempfile +import shutil +import getopt +import glob +import sys +import os + +HELP = """\ +Usage: mgarepo getsrpm [OPTIONS] REPPKGURL + +Generates the source RPM (.srpm) file of a given package. + +Options: + -c Use files in current/ directory (default) + -p Use files in pristine/ directory + -v VER Use files from the version specified by VER (e.g. 2.2.1-2cl) + -r REV Use files from current directory, in revision REV (e.g. 1001) + -t DIR Put SRPM file in directory DIR when done (default is ".") + -P USER Define the RPM packager inforamtion to USER + -s FILE Run script with "FILE TOPDIR SPECFILE" command + -n Rename the package to include the revision number + -l Use subversion log to build rpm %changelog + -T FILE Template to be used to generate the %changelog + -M Do not use the mirror (use the main repository) + -h Show this message + -S Do not download sources from the binary repository + --check Check integrity of files fetched from the binary repository + --strict Check if the given revision contains changes in REPPKGURL + +Examples: + mgarepo getsrpm python + mgarepo getsrpm -l python + mgarepo getsrpm http://foo.bar/svn/cnc/snapshot/python + mgarepo getsrpm -p http://foo.bar/svn/cnc/releases/8cl/python + mgarepo getsrpm -r 1001 file:///svn/cnc/snapshot/python +""" + +def mode_callback(option, opt, val, parser, mode): + opts = parser.values + opts.mode = mode + if mode == "version": + try: + opts.version, opts.release = val.split("-", 1) + except ValueError: + raise Error, "wrong version, use something like 2.2-1mdk" + elif mode == "revision": + opts.revision = val + +def parse_options(): + parser = OptionParser(help=HELP) + parser.defaults["mode"] = "current" + parser.defaults["version"] = None + parser.defaults["release"] = None + parser.defaults["revision"] = None + parser.defaults["submit"] = False + callback_options = dict(action="callback", callback=mode_callback, + type="string", dest="__ignore") + parser.add_option("-c", callback_kwargs={"mode": "current"}, nargs=0, + **callback_options) + parser.add_option("-p", callback_kwargs={"mode": "pristine"}, nargs=0, + **callback_options) + parser.add_option("-r", callback_kwargs={"mode": "revision"}, nargs=1, + **callback_options) + parser.add_option("-v", callback_kwargs={"mode": "version"}, nargs=1, + **callback_options) + parser.add_option("-t", dest="targetdirs", action="append", default=[]) + parser.add_option("-s", dest="scripts", action="append", default=[]) + parser.add_option("-P", dest="packager", default="") + parser.add_option("-n", dest="revname", action="store_true") + parser.add_option("-l", dest="svnlog", action="store_true") + parser.add_option("-T", dest="template", type="string", default=None) + parser.add_option("-S", dest="use_binrepo", default=True, + action="store_false") + parser.add_option("--check", dest="binrepo_check", default=False, + action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + parser.add_option("--strict", dest="strict", default=False, + action="store_true") + opts, args = parser.parse_args() + del opts.__ignore + if len(args) != 1: + raise Error, "invalid arguments" + opts.pkgdirurl = package_url(args[0]) + opts.verbose = 1 + return opts + +def main(): + do_command(parse_options, get_srpm) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/log.py b/MgaRepo/commands/log.py new file mode 100644 index 0000000..69058b9 --- /dev/null +++ b/MgaRepo/commands/log.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +from MgaRepo import config, mirror, disable_mirror +from MgaRepo.command import * +from MgaRepo.layout import package_url, checkout_url +from MgaRepo.rpmutil import sync +from MgaRepo.util import execcmd +import sys +import os + +HELP = """\ +Usage: mgarepo log [OPTIONS] [PACKAGE] + +Shows the SVN log for a given package. + +Options: + -h Show this message + -v Show changed paths + -l LIMIT Limit of log entries to show + -r REV Show a specific revision + -M Do not use the mirror (use the main repository) + +Examples: + mgarepo log mutt + mgarepo log 2009.1/mutt +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-v", dest="verbose", action="store_true", + default=False) + parser.add_option("-l", "--limit", dest="limit", type="int", + default=None) + parser.add_option("-r", dest="revision", type="string", default=None) + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + opts, args = parser.parse_args() + if len(args): + opts.pkgdirurl = package_url(args[0]) + else: + parser.error("log requires a package name") + return opts + +def svn_log(pkgdirurl, verbose=False, limit=None, revision=None): + mirror.info(pkgdirurl) + url = checkout_url(pkgdirurl) + svncmd = config.get("global", "svn-command", "svn") + args = [svncmd, "log", url] + if verbose: + args.append("-v") + if limit: + args.append("-l") + args.append(limit) + if revision: + args.append("-r") + args.append(revision) + if os.isatty(sys.stdin.fileno()): + args.append("| less") + rawcmd = " ".join(args) + execcmd(rawcmd, show=True) + +def main(): + do_command(parse_options, svn_log) diff --git a/MgaRepo/commands/markrelease.py b/MgaRepo/commands/markrelease.py new file mode 100644 index 0000000..857d38d --- /dev/null +++ b/MgaRepo/commands/markrelease.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +# +# This program will append a release to the Conectiva Linux package +# repository system. It's meant to be a startup system to include +# pre-packaged SRPMS in the repository, thus, you should not commit +# packages over an ongoing package structure (with changes in current/ +# directory and etc). Also, notice that packages must be included in +# cronological order. +# +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.simplerpm import SRPM +from MgaRepo.rpmutil import mark_release +from MgaRepo.util import get_auth +import getopt +import sys +import os + +HELP = """\ +*** WARNING --- You probably SHOULD NOT use this program! --- WARNING *** + +Usage: mgarepo markrelease [OPTIONS] REPPKGURL + +This subcommand creates a 'tag' for a given revision of a given package. + +The tag will be stored in the directory releases/ inside the package +structure. + +Options: + -f FILE Try to extract information from given file + -r REV Revision which will be used to make the release copy tag + -v VER Version which will be used to make the release copy tag + -n Append package name to provided URL + -h Show this message + +Examples: + mgarepo markrelease -r 68 -v 1.0-1 file://svn/cnc/snapshot/foo + mgarepo markrelease -f @68:foo-1.0-1.src.rpm file://svn/cnc/snapshot/foo + mgarepo markrelease -r 68 -f foo-1.0.src.rpm file://svn/cnc/snapshot/foo +""" + +def version_callback(option, opt, val, parser): + opts = parser.values + try: + opts.version, opts.release = val.split("-", 1) + except ValueError: + raise Error, "wrong version, use something like 1:2.2-1mdk" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.defaults["version"] = None + parser.defaults["release"] = None + parser.add_option("-v", action="callback", callback=version_callback, + nargs=1, type="string", dest="__ignore") + parser.add_option("-r", dest="revision") + parser.add_option("-f", dest="filename") + parser.add_option("-n", dest="appendname", action="store_true") + opts, args = parser.parse_args() + + if len(args) != 1: + raise Error, "invalid arguments" + + opts.pkgdirurl = package_url(args[0], mirrored=False) + + filename = opts.filename + appendname = opts.appendname + del opts.filename, opts.appendname, opts.__ignore + + if filename: + if not os.path.isfile(filename): + raise Error, "file not found: "+filename + if not opts.revision: + basename = os.path.basename(filename) + end = basename.find(":") + if basename[0] != "@" or end == -1: + raise Error, "couldn't guess revision from filename" + opts.revision = basename[1:end] + srpm = None + if not opts.version: + srpm = SRPM(filename) + if srpm.epoch: + opts.version = "%s:%s" % (srpm.epoch, srpm.version) + else: + opts.version = srpm.version + opts.release = srpm.release + if appendname: + if not srpm: + srpm = SRPM(filename) + opts.pkgdirurl = "/".join([opts.pkgdirurl, srpm.name]) + elif appendname: + raise Error, "option -n requires option -f" + elif not opts.revision: + raise Error, "no revision provided" + elif not opts.version: + raise Error, "no version provided" + #get_auth() + return opts + +def main(): + do_command(parse_options, mark_release) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/patchspec.py b/MgaRepo/commands/patchspec.py new file mode 100644 index 0000000..9a5811c --- /dev/null +++ b/MgaRepo/commands/patchspec.py @@ -0,0 +1,38 @@ +#!/usr/bin/python +# +# This program will try to patch a spec file from a given package url. +# +from MgaRepo import Error +from MgaRepo.rpmutil import patch_spec +from MgaRepo.command import * +from MgaRepo.layout import package_url +import getopt +import sys + +HELP = """\ +Usage: mgarepo patchspec [OPTIONS] REPPKGURL PATCHFILE + +It will try to patch a spec file from a given package url. + +Options: + -l LOG Use LOG as log message + -h Show this message + +Examples: + mgarepo patchspec http://repos/svn/cnc/snapshot/foo +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-l", dest="log", default="") + opts, args = parser.parse_args() + if len(args) != 2: + raise Error, "invalid arguments" + opts.pkgdirurl = package_url(args[0], mirrored=False) + opts.patchfile = args[1] + return opts + +def main(): + do_command(parse_options, patch_spec) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/putsrpm.py b/MgaRepo/commands/putsrpm.py new file mode 100644 index 0000000..68d87d0 --- /dev/null +++ b/MgaRepo/commands/putsrpm.py @@ -0,0 +1,59 @@ +#!/usr/bin/python +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.layout import package_url +from MgaRepo.rpmutil import put_srpm +import getopt +import sys, os + +HELP = """\ +Usage: mgarepo putsrpm [OPTIONS] SOURCERPMS + +Will import source RPMs into the SVN repository. + +If the package was already imported, it will add the new files and remove +those not present in the source RPM. + +Options: + -m LOG Log message used when commiting changes + -t Create version-release tag on releases/ + -b NAME The distribution branch to place it + -d URL The URL of base directory where packages will be placed + -c URL The URL of the base directory where the changelog will be + placed + -s Don't strip the changelog from the spec + (nor import it into misc/) + -n Don't try to rename the spec file + -h Show this message + +Examples: + mgarepo putsrpm pkg/SRPMS/pkg-2.0-1.src.rpm + mgarepo putsrpm -b 2009.1 foo-1.1-1.src.rpm +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-l", dest="logmsg", default="") + parser.add_option("-t", dest="markrelease", action="store_true", + default=False) + parser.add_option("-s", dest="striplog", action="store_false", + default=True) + parser.add_option("-b", dest="branch", type="string", default=None) + parser.add_option("-d", dest="baseurl", type="string", default=None) + parser.add_option("-c", dest="baseold", type="string", default=None) + parser.add_option("-n", dest="rename", action="store_false", + default=True) + opts, args = parser.parse_args() + opts.srpmfiles = args + return opts + +def put_srpm_cmd(srpmfiles, markrelease=False, striplog=True, branch=None, + baseurl=None, baseold=None, logmsg=None, rename=False): + for path in srpmfiles: + put_srpm(path, markrelease, striplog, branch, baseurl, baseold, + logmsg, rename) + +def main(): + do_command(parse_options, put_srpm_cmd) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/rpmlog.py b/MgaRepo/commands/rpmlog.py new file mode 100644 index 0000000..ae8997d --- /dev/null +++ b/MgaRepo/commands/rpmlog.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# +# This program will convert the output of "svn log" to be suitable +# for usage in an rpm %changelog session. +# +from MgaRepo import Error, layout, disable_mirror +from MgaRepo.command import * +from MgaRepo.svn import SVN +from MgaRepo.log import get_changelog, split_spec_changelog +from cStringIO import StringIO +import getopt +import os +import sys + +HELP = """\ +Usage: mgarepo rpmlog [OPTIONS] REPPKGDIRURL + +Prints the RPM changelog of a given package. + +Options: + -r REV Collect logs from given revision to revision 0 + -n NUM Output only last NUM entries + -T FILE %changelog template file to be used + -o Append old package changelog + -p Append changelog found in .spec file + -s Sort changelog entries, even from the old log + -M Do not use the mirror (use the main repository) + -h Show this message + +Examples: + mgarepo rpmlog python + mgarepo rpmlog http://svn.mandriva.com/svn/packages/cooker/python +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-r", dest="revision") + parser.add_option("-n", dest="size", type="int") + parser.add_option("-T", "--template", dest="template", type="string") + parser.add_option("-o", dest="oldlog", default=False, + action="store_true") + parser.add_option("-p", dest="usespec", default=False, + action="store_true") + parser.add_option("-s", dest="sort", default=False, + action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + opts, args = parser.parse_args() + if len(args) != 1: + raise Error, "invalid arguments" + opts.pkgdirurl = layout.package_url(args[0]) + return opts + +def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort): + another = None + if usespec: + svn = SVN() + specurl = layout.package_spec_url(pkgdirurl) + rawspec = svn.cat(specurl, rev=revision) + spec, another = split_spec_changelog(StringIO(rawspec)) + newlog = get_changelog(pkgdirurl, another=another, rev=revision, + size=size, sort=sort, template=template, oldlog=oldlog) + sys.stdout.writelines(newlog) + +def main(): + do_command(parse_options, rpmlog) + +# vim:sw=4:ts=4:et diff --git a/MgaRepo/commands/submit.py b/MgaRepo/commands/submit.py new file mode 100644 index 0000000..79afd21 --- /dev/null +++ b/MgaRepo/commands/submit.py @@ -0,0 +1,211 @@ +#!/usr/bin/python +from MgaRepo import Error, config, layout, mirror +from MgaRepo.svn import SVN +from MgaRepo.command import * +from MgaRepo.rpmutil import get_spec, get_submit_info +from MgaRepo.util import get_auth, execcmd, get_helper +import urllib +import getopt +import sys +import re +import subprocess +import uuid + +import xmlrpclib + +HELP = """\ +Usage: mgarepo submit [OPTIONS] [URL[@REVISION] ...] + +Submits the package from URL to the submit host. + +The submit host will try to build the package, and upon successful +completion will 'tag' the package and upload it to the official +repositories. + +The package name can refer to an alias to a group of packages defined in +the section submit-groups of the configuration file. + +The status of the submit can visualized at: + +http://kenobi.mandriva.com/bs/output.php + +If no URL and revision are specified, the latest changed revision in the +package working copy of the current directory will be used. + +Options: + -t TARGET Submit given package URL to given target + -l Just list available targets + -r REV Provides a revision number (when not providing as an + argument) + -s The host in which the package URL will be submitted + (defaults to the host in the URL) + -a Submit all URLs at once (depends on server-side support) + -i SID Use the submit identifier SID + -h Show this message + --distro The distribution branch where the packages come from + --define Defines one variable to be used by the submit scripts + in the submit host + +Examples: + mgarepo submit + mgarepo submit foo + mgarepo submit 1/foo + mgarepo submit foo@14800 bar baz@11001 + mgarepo submit https://repos/svn/mga/cauldron/foo + mgarepo submit -l https://repos + mgarepo submit 1/my-packages@11011 + mgarepo submit --define section=core/testing -t 1 +""" + +DEFAULT_TARGET = "Cooker" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.defaults["revision"] = None + parser.add_option("-t", dest="target", default=None) + parser.add_option("-l", action="callback", callback=list_targets) + parser.add_option("-r", dest="revision", type="string", nargs=1) + parser.add_option("-s", dest="submithost", type="string", nargs=1, + default=None) + parser.add_option("-i", dest="sid", type="string", nargs=1, + default=None) + parser.add_option("-a", dest="atonce", action="store_true", default=False) + parser.add_option("--distro", dest="distro", type="string", + default=None) + parser.add_option("--define", action="append", default=[]) + opts, args = parser.parse_args() + if not args: + name, url, rev = get_submit_info(".") + args = ["%s@%s" % (url, str(rev))] + print "Submitting %s at revision %s" % (name, rev) + print "URL: %s" % url + if opts.revision is not None: + # backwards compatibility with the old -r usage + if len(args) == 1: + args[0] = args[0] + "@" + opts.revision + else: + raise Error, "can't use -r REV with more than one package name" + del opts.revision + if len(args) == 2: + # prevent from using the old <name> <rev> syntax + try: + rev = int(args[1]) + except ValueError: + # ok, it is a package name, let it pass + pass + else: + raise Error, "the format <name> <revision> is deprecated, "\ + "use <name>@<revision> instead" + # expand group aliases + expanded = [] + for nameurl in args: + expanded.extend(expand_group(nameurl)) + if expanded != args: + print "Submitting: %s" % " ".join(expanded) + args = expanded + # generate URLs for package names: + opts.urls = [mirror.strip_username( + layout.package_url(nameurl, distro=opts.distro, mirrored=False)) + for nameurl in args] + # find the revision if not specified: + newurls = [] + for url in opts.urls: + if not "@" in url: + print "Fetching revision..." + courl = layout.checkout_url(url) + log = SVN().log(courl, limit=1) + if not log: + raise Error, "can't find a revision for %s" % courl + ci = log[0] + print "URL:", url + print "Commit:", + print "%d | %s" % (ci.revision, ci.author), + if ci.lines: + line = " ".join(ci.lines).strip() + if len(line) > 57: + line = line[:57] + "..." + print "| %s" % line, + print + url = url + "@" + str(ci.revision) + newurls.append(url) + opts.urls[:] = newurls + # choose a target if not specified: + if opts.target is None and opts.distro is None: + target = layout.distro_branch(opts.urls[0]) or DEFAULT_TARGET + print "Implicit target: %s" % target + opts.target = target + del opts.distro + return opts + +def expand_group(group): + name, rev = layout.split_url_revision(group) + distro = None + if "/" in name: + distro, name = name.rsplit("/", 1) + found = config.get("submit-groups", name) + packages = [group] + if found: + packages = found.split() + if rev: + packages = [("%s@%s" % (package, rev)) + for package in packages] + if distro: + packages = ["%s/%s" % (distro, package) + for package in packages] + return packages + +def list_targets(option, opt, val, parser): + host = config.get("submit", "host") + if host is None: + raise Error, "no submit host defined in mgarepo.conf" + createsrpm = get_helper("create-srpm") + #TODO make it configurable + command = "ssh %s %s --list" % (host, createsrpm) + execcmd(command, show=True) + sys.exit(0) + +def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): + if submithost is None: + submithost = config.get("submit", "host") + if submithost is None: + # extract the submit host from the svn host + type, rest = urllib.splittype(pkgdirurl) + host, path = urllib.splithost(rest) + user, host = urllib.splituser(host) + submithost, port = urllib.splitport(host) + del type, user, port, path, rest + # runs a create-srpm in the server through ssh, which will make a + # copy of the rpm in the export directory + createsrpm = get_helper("create-srpm") + baseargs = ["ssh", submithost, createsrpm, "-t", target] + if not sid: + sid = uuid.uuid4() + define.append("sid=%s" % sid) + for entry in reversed(define): + baseargs.append("--define") + baseargs.append(entry) + cmdsargs = [] + if len(urls) == 1: + # be compatible with server-side mgarepo versions older than 1.6.90 + url, rev = layout.split_url_revision(urls[0]) + baseargs.append("-r") + baseargs.append(str(rev)) + baseargs.append(url) + cmdsargs.append(baseargs) + elif atonce: + cmdsargs.append(baseargs + urls) + else: + cmdsargs.extend((baseargs + [url]) for url in urls) + for cmdargs in cmdsargs: + command = subprocess.list2cmdline(cmdargs) + status, output = execcmd(command) + if status == 0: + print "Package submitted!" + else: + sys.stderr.write(output) + sys.exit(status) + +def main(): + do_command(parse_options, submit) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/commands/switch.py b/MgaRepo/commands/switch.py new file mode 100644 index 0000000..ccca76e --- /dev/null +++ b/MgaRepo/commands/switch.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +from MgaRepo.command import * +from MgaRepo.rpmutil import switch + +HELP = """\ +Usage: mgarepo switch [URL] + +Relocates the working copy to the base location URL. + +If URL is not provided, it will use the option repository from mgarepo.conf +as default, or, if the current working copy is already based in +default_parent, it will use the location from the mirror option from +mgarepo.conf. + +If the current work is based in another URL, it will use default_parent. + +Options: + -h Show this message + +Examples: + mgarepo switch + mgarepo switch https://mirrors.localnetwork/svn/packages/ +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + if len(args): + opts.mirrorurl = args[0] + return opts + +def main(): + do_command(parse_options, switch) diff --git a/MgaRepo/commands/sync.py b/MgaRepo/commands/sync.py new file mode 100644 index 0000000..1198f3f --- /dev/null +++ b/MgaRepo/commands/sync.py @@ -0,0 +1,38 @@ +#!/usr/bin/python +from MgaRepo.command import * +from MgaRepo.rpmutil import sync + +HELP = """\ +Usage: mgarepo sync + +Will add or remove from the working copy those files added or removed +in the spec file. + +It will not commit the changes. + +Options: + -c Commit the changes, as in ci + --dry-run Print results without changing the working copy + --download -d + Try to download the source files not found + -h Show this message + +Examples: + mgarepo sync +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("--dry-run", dest="dryrun", default=False, + action="store_true") + parser.add_option("-c", dest="ci", default=False, + action="store_true") + parser.add_option("-d", "--download", dest="download", default=False, + action="store_true") + opts, args = parser.parse_args() + if len(args): + opts.target = args[0] + return opts + +def main(): + do_command(parse_options, sync) diff --git a/MgaRepo/commands/up.py b/MgaRepo/commands/up.py new file mode 100644 index 0000000..646a79f --- /dev/null +++ b/MgaRepo/commands/up.py @@ -0,0 +1,22 @@ +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.rpmutil import update + +HELP = """\ +Usage: mgarepo up [PATH] + +Update the package working copy and synchronize all binaries. + +Options: + -h help +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + if args: + opts.target = args[0] + return opts + +def main(): + do_command(parse_options, update) diff --git a/MgaRepo/commands/upload.py b/MgaRepo/commands/upload.py new file mode 100644 index 0000000..dfcae5d --- /dev/null +++ b/MgaRepo/commands/upload.py @@ -0,0 +1,28 @@ +from MgaRepo import Error +from MgaRepo.command import * +from MgaRepo.rpmutil import upload + +HELP = """\ +Usage: mgarepo upload [OPTIONS] [PATH] + +Upload a given file to the binary sources repository. + +It will also update the contents of the 'binrepo.lst' file and leave it +uncommited. + +If the path is a directory, all the contents of the directory will be +uploaded or removed. + +Options: + -h help + +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + opts.paths = args + return opts + +def main(): + do_command(parse_options, upload) diff --git a/MgaRepo/layout.py b/MgaRepo/layout.py new file mode 100644 index 0000000..f0aaf27 --- /dev/null +++ b/MgaRepo/layout.py @@ -0,0 +1,207 @@ +""" Handles repository layout scheme and package URLs.""" + +import os +import urlparse + +from MgaRepo import Error, config +from MgaRepo.svn import SVN + +__all__ = ["package_url", "checkout_url", "repository_url", "get_url_revision"] + +def layout_dirs(): + devel_branch = config.get("global", "trunk-dir", "cauldron/") + devel_branch = os.path.normpath(devel_branch) + branches_dir = config.get("global", "branches-dir", "updates/") + branches_dir = os.path.normpath(branches_dir) + return devel_branch, branches_dir + +def get_url_revision(url, retrieve=True): + """Get the revision from a given URL + + If the URL contains an explicit revision number (URL@REV), just use it + without even checking if the revision really exists. + + The parameter retrieve defines whether it must ask the SVN server for + the revision number or not when it is not found in the URL. + """ + url, rev = split_url_revision(url) + if rev is None and retrieve: + # if no revspec was found, ask the server + svn = SVN() + rev = svn.revision(url) + return rev + +def unsplit_url_revision(url, rev): + if rev is None: + newurl = url + else: + parsed = list(urlparse.urlparse(url)) + path = os.path.normpath(parsed[2]) + parsed[2] = path + "@" + str(rev) + newurl = urlparse.urlunparse(parsed) + return newurl + +def split_url_revision(url): + """Returns a tuple (url, rev) from an subversion URL with @REV + + If the revision is not present in the URL, rev is None. + """ + parsed = list(urlparse.urlparse(url)) + path = os.path.normpath(parsed[2]) + dirs = path.rsplit("/", 1) + lastname = dirs[-1] + newname = lastname + index = lastname.rfind("@") + rev = None + if index != -1: + newname = lastname[:index] + rawrev = lastname[index+1:] + if rawrev: + try: + rev = int(rawrev) + if rev < 0: + raise ValueError + except ValueError: + raise Error, "invalid revision specification on URL: %s" % url + dirs[-1] = newname + newpath = "/".join(dirs) + parsed[2] = newpath + newurl = urlparse.urlunparse(parsed) + return newurl, rev + +def checkout_url(pkgdirurl, branch=None, version=None, release=None, + releases=False, pristine=False, append_path=None): + """Get the URL of a branch of the package, defaults to current/ + + It tries to preserve revisions in the format @REV. + """ + parsed = list(urlparse.urlparse(pkgdirurl)) + path, rev = split_url_revision(parsed[2]) + if releases: + path = os.path.normpath(path + "/releases") + elif version: + assert release is not None + path = os.path.normpath(path + "/releases/" + version + "/" + release) + elif pristine: + path = os.path.join(path, "pristine") + elif branch: + path = os.path.join(path, "branches", branch) + else: + path = os.path.join(path, "current") + if append_path: + path = os.path.join(path, append_path) + path = unsplit_url_revision(path, rev) + parsed[2] = path + newurl = urlparse.urlunparse(parsed) + return newurl + +def convert_default_parent(url): + """Removes the cauldron/ component from the URL""" + parsed = list(urlparse.urlparse(url)) + path = os.path.normpath(parsed[2]) + rest, last = os.path.split(path) + parsed[2] = rest + newurl = urlparse.urlunparse(parsed) + return newurl + +def remove_current(pkgdirurl): + parsed = list(urlparse.urlparse(pkgdirurl)) + path = os.path.normpath(parsed[2]) + rest, last = os.path.split(path) + if last == "current": + # FIXME this way we will not allow packages to be named "current" + path = rest + parsed[2] = path + newurl = urlparse.urlunparse(parsed) + return newurl + +def repository_url(mirrored=False): + url = None + if mirrored and config.getbool("global", "use-mirror", "yes"): + url = config.get("global", "mirror") + if url is None: + url = config.get("global", "repository") + if not url: + # compatibility with the default_parent configuration option + default_parent = config.get("global", "default_parent") + if default_parent is None: + raise Error, "you need to set the 'repository' " \ + "configuration option on mgarepo.conf" + url = convert_default_parent(default_parent) + return url + +def package_url(name_or_url, version=None, release=None, distro=None, + mirrored=True): + """Returns a tuple with the absolute package URL and its name + + @name_or_url: name, relative path, or URL of the package. In case it is + a URL, the URL will just be 'normalized'. + @version: the version to be fetched from releases/ (requires release) + @release: the release number to be fetched from releases/$version/ + @distro: the name of the repository branch inside updates/ + @mirrored: return an URL based on the mirror repository, if enabled + """ + from MgaRepo import mirror + if "://" in name_or_url: + pkgdirurl = mirror.normalize_path(name_or_url) + pkgdirurl = remove_current(pkgdirurl) + if mirror.using_on(pkgdirurl) and not mirrored: + pkgdirurl = mirror.relocate_path(mirror.mirror_url(), + repository_url(), pkgdirurl) + else: + name = name_or_url + devel_branch, branches_dir = layout_dirs() + if distro or "/" in name: + default_branch = branches_dir + if distro: + default_branch = os.path.join(default_branch, distro) + else: + default_branch = devel_branch # cauldron + path = os.path.join(default_branch, name) + parsed = list(urlparse.urlparse(repository_url(mirrored=mirrored))) + parsed[2] = os.path.join(parsed[2], path) + pkgdirurl = urlparse.urlunparse(parsed) + return pkgdirurl + +def package_name(pkgdirurl): + """Returns the package name from a package URL + + It takes care of revision numbers""" + parsed = urlparse.urlparse(pkgdirurl) + path, rev = split_url_revision(parsed[2]) + rest, name = os.path.split(path) + return name + +def package_spec_url(pkgdirurl, *args, **kwargs): + """Returns the URL of the specfile of a given package URL + + The parameters are the same used by checkout_url, except append_path. + """ + kwargs["append_path"] = "SPECS/" + package_name(pkgdirurl) + ".spec" + specurl = checkout_url(pkgdirurl, *args, **kwargs) + return specurl + +def distro_branch(pkgdirurl): + """Tries to guess the distro branch name from a package URL""" + from MgaRepo.mirror import same_base + found = None + repo = repository_url() + if same_base(repo, pkgdirurl): + devel_branch, branches_dir = layout_dirs() + repo_path = urlparse.urlparse(repo)[2] + devel_path = os.path.join(repo_path, devel_branch) + branches_path = os.path.join(repo_path, branches_dir) + parsed = urlparse.urlparse(pkgdirurl) + path = os.path.normpath(parsed[2]) + if path.startswith(devel_path): + # devel_branch must be before branches_dir in order to allow + # devel_branch to be inside branches_dir, as in /branches/cooker + _, found = os.path.split(devel_branch) + elif path.startswith(branches_path): + comps = path.split("/") + if branches_path == "/": + found = comps[1] + elif len(comps) >= 2: # must be at least branch/pkgname + found = comps[branches_path.count("/")+1] + return found + diff --git a/MgaRepo/log.py b/MgaRepo/log.py new file mode 100644 index 0000000..56965c5 --- /dev/null +++ b/MgaRepo/log.py @@ -0,0 +1,633 @@ +#!/usr/bin/python +from MgaRepo import Error, config, layout +from MgaRepo.svn import SVN +from MgaRepo.util import execcmd + +try: + from Cheetah.Template import Template +except ImportError: + raise Error, "mgarepo requires the package python-cheetah" + +from cStringIO import StringIO + +import sys +import os +import re +import time +import locale +import glob +import tempfile +import shutil +import subprocess + + +locale.setlocale(locale.LC_ALL, "C") + +default_template = """ +#if not $releases_by_author[-1].visible + ## Hide the first release that contains no changes. It must be a + ## reimported package and the log gathered from misc/ already should + ## contain a correct entry for the version-release: + #set $releases_by_author = $releases_by_author[:-1] +#end if +#for $rel in $releases_by_author +* $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release ++ Revision: $rel.revision +## #if not $rel.released +##+ Status: not released +## #end if + #if not $rel.visible ++ rebuild (emptylog) + #end if + #for $rev in $rel.release_revisions + #for $line in $rev.lines +$line + #end for + #end for + + #for $author in $rel.authors + #if not $author.visible + #continue + #end if + ##alternatively, one could use: + ###if $author.email == "root" + ## #continue + ###end if + + $author.name <$author.email> + #for $rev in $author.revisions + #for $line in $rev.lines + $line + #end for + #end for + + #end for +#end for +""" + +def getrelease(pkgdirurl, rev=None, macros=[], exported=None): + """Tries to obtain the version-release of the package for a + yet-not-markrelease revision of the package. + + Is here where things should be changed if "automatic release increasing" + will be used. + """ + from MgaRepo.rpmutil import rpm_macros_defs + svn = SVN() + pkgcurrenturl = os.path.join(pkgdirurl, "current") + specurl = os.path.join(pkgcurrenturl, "SPECS") + if exported is None: + tmpdir = tempfile.mktemp() + svn.export(specurl, tmpdir, rev=rev) + else: + tmpdir = os.path.join(exported, "SPECS") + try: + found = glob.glob(os.path.join(tmpdir, "*.spec")) + if not found: + raise Error, "no .spec file found inside %s" % specurl + specpath = found[0] + options = rpm_macros_defs(macros) + command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " + "--specfile %s %s") % + (specpath, options)) + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + pipe.wait() + output = pipe.stdout.read() + error = pipe.stderr.read() + if pipe.returncode != 0: + raise Error, "Error in command %s: %s" % (command, error) + releases = output.split() + try: + epoch, vr = releases[0].split(":", 1) + version, release = vr.split("-", 1) + except ValueError: + raise Error, "Invalid command output: %s: %s" % \ + (command, output) + #XXX check if this is the right way: + if epoch == "(none)": + ev = version + else: + ev = epoch + ":" + version + return ev, release + finally: + if exported is None and os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + +class _Revision: + lines = [] + date = None + raw_date = None + revision = None + author_name = None + author_email = None + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __repr__(self): + lines = repr(self.lines)[:30] + "...]" + line = "<_Revision %d author=%r date=%r lines=%s>" % \ + (self.revision, self.author, self.date, lines) + return line + + +class _Release(_Revision): + version = None + release = None + revisions = [] + release_revisions = [] + authors = [] + visible = False + + def __init__(self, **kwargs): + self.revisions = [] + _Revision.__init__(self, **kwargs) + + def __repr__(self): + line = "<_Release v=%s r=%s revs=%r>" % \ + (self.version, self.release, self.revisions) + return line + +unescaped_macro_pat = re.compile(r"([^%])%([^%])") + +def escape_macros(text): + escaped = unescaped_macro_pat.sub("\\1%%\\2", text) + return escaped + +def format_lines(lines): + first = 1 + entrylines = [] + perexpr = re.compile(r"([^%])%([^%])") + for line in lines: + if line: + line = escape_macros(line) + if first: + first = 0 + line = line.lstrip() + if line[0] != "-": + nextline = "- " + line + else: + nextline = line + elif line[0] != " " and line[0] != "-": + nextline = " " + line + else: + nextline = line + if nextline not in entrylines: + entrylines.append(nextline) + return entrylines + + +class _Author: + name = None + email = None + revisions = None + visible = False + + +def group_releases_by_author(releases): + allauthors = [] + grouped = [] + for release in releases: + + # group revisions of the release by author + authors = {} + latest = None + for revision in release.revisions: + authors.setdefault(revision.author, []).append(revision) + + # create _Authors and sort them by their latest revisions + decorated = [] + for authorname, revs in authors.iteritems(): + author = _Author() + author.name = revs[0].author_name + author.email = revs[0].author_email + author.revisions = revs + # #41117: mark those authors without visible messages + author.visible = bool(sum(len(rev.lines) for rev in revs)) + revlatest = author.revisions[0] + # keep the latest revision even for completely invisible + # authors (below) + if latest is None or revlatest.revision > latest.revision: + latest = revlatest + if not author.visible: + # only sort those visible authors, invisible ones are used + # only in "latest" + continue + decorated.append((revlatest.revision, author)) + decorated.sort(reverse=1) + + if release.visible: + release.authors = [t[1] for t in decorated] + firstrel, release.authors = release.authors[0], release.authors[1:] + release.author_name = firstrel.name + release.author_email = firstrel.email + release.release_revisions = firstrel.revisions + else: + # we don't care about other possible authors in completely + # invisible releases + firstrev = release.revisions[0] + release.author_name = firstrev.author_name + release.author_email = firstrev.author_email + release.raw_date = firstrev.raw_date + release.date = firstrev.date + + release.date = latest.date + release.raw_date = latest.raw_date + release.revision = latest.revision + + grouped.append(release) + + return grouped + + +def group_revisions_by_author(currentlog): + revisions = [] + last_author = None + for entry in currentlog: + revision = _Revision() + revision.lines = format_lines(entry.lines) + revision.raw_date = entry.date + revision.date = parse_raw_date(entry.date) + revision.revision = entry.revision + if entry.author == last_author: + revisions[-1].revisions.append(revision) + else: + author = _Author() + author.name, author.email = get_author_name(entry.author) + author.revisions = [revision] + revisions.append(author) + last_author = entry.author + return revisions + + +emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>") + +def get_author_name(author): + found = emailpat.match(config.get("users", author, author)) + name = ((found and found.group("name")) or author) + email = ((found and found.group("email")) or author) + return name, email + +def parse_raw_date(rawdate): + return time.strftime("%a %b %d %Y", rawdate) + +def filter_log_lines(lines): + # Lines in commit messages beginning with CLOG will be the only shown + # in the changelog. These lines will have the CLOG token and blanks + # stripped from the beginning. + onlylines = None + clogstr = config.get("log", "unignore-string") + if clogstr: + clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr) + onlylines = [clogre.sub("", line) + for line in lines if line.startswith(clogstr)] + if onlylines: + filtered = onlylines + else: + # Lines in commit messages containing SILENT at any position will be + # skipped; commits with their log messages beggining with SILENT in the + # first positionj of the first line will have all lines ignored. + ignstr = config.get("log", "ignore-string", "SILENT") + if len(lines) and lines[0].startswith(ignstr): + return [] + filtered = [line for line in lines if ignstr not in line] + return filtered + + +def make_release(author=None, revision=None, date=None, lines=None, + entries=[], released=True, version=None, release=None): + rel = _Release() + rel.author = author + if author: + rel.author_name, rel.author_email = get_author_name(author) + rel.revision = revision + rel.version = version + rel.release = release + rel.date = (date and parse_raw_date(date)) or None + rel.lines = lines + rel.released = released + rel.visible = False + for entry in entries: + lines = filter_log_lines(entry.lines) + revision = _Revision() + revision.revision = entry.revision + revision.lines = format_lines(lines) + if revision.lines: + rel.visible = True + revision.date = parse_raw_date(entry.date) + revision.raw_date = entry.date + revision.author = entry.author + (revision.author_name, revision.author_email) = \ + get_author_name(entry.author) + rel.revisions.append(revision) + return rel + + +def dump_file(releases, currentlog=None, template=None): + templpath = template or config.get("template", "path", + "/usr/share/mgarepo/default.chlog") + params = {} + if templpath is None or not os.path.exists(templpath): + params["source"] = default_template + sys.stderr.write("warning: %s not found. using built-in template.\n"% + templpath) + else: + params["file"] = templpath + releases_author = group_releases_by_author(releases) + revisions_author = group_revisions_by_author(currentlog) + params["searchList"] = [{"releases_by_author" : releases_author, + "releases" : releases, + "revisions_by_author": revisions_author}] + t = Template(**params) + return t.respond() + + +class InvalidEntryError(Exception): + pass + +def parse_repsys_entry(revlog): + # parse entries in the format: + # %repsys <operation> + # key: value + # .. + # <newline> + # <comments> + # + if len(revlog.lines) == 0 or not revlog.lines[0].startswith("%repsys"): + raise InvalidEntryError + try: + data = {"operation" : revlog.lines[0].split()[1]} + except IndexError: + raise InvalidEntryError + for line in revlog.lines[1:]: + if not line: + break + try: + key, value = line.split(":", 1) + except ValueError: + raise InvalidEntryError + data[key.strip().lower()] = value.strip() # ??? + return data + + +def get_revision_offset(): + try: + revoffset = config.getint("log", "revision-offset", 0) + except (ValueError, TypeError): + raise Error, ("Invalid revision-offset number in configuration " + "file(s).") + return revoffset or 0 + +oldmsgpat = re.compile( + r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.") + +def parse_markrelease_log(relentry): + if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \ + or parse_repsys_entry(relentry))): + raise InvalidEntryError + from_rev = None + path = None + for changed in relentry.changed: + if changed["action"] == "A" and changed["from_rev"]: + from_rev = changed["from_rev"] + path = changed["path"] + break + else: + raise InvalidEntryError + # get the version and release from the names in the path, do not relay + # on log messages + version, release = path.rsplit(os.path.sep, 3)[-2:] + return version, release, from_rev + + +def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, + template=None, macros=[], exported=None): + concat = config.get("log", "concat", "").split() + revoffset = get_revision_offset() + svn = SVN() + pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True) + pkgcurrenturl = layout.checkout_url(pkgdirurl) + releaseslog = svn.log(pkgreleasesurl, noerror=1) + currentlog = svn.log(pkgcurrenturl, limit=size, start=rev, + end=revoffset) + + # sort releases by copyfrom-revision, so that markreleases for same + # revisions won't look empty + releasesdata = [] + if releaseslog: + for relentry in releaseslog[::-1]: + try: + (version, release, relrevision) = \ + parse_markrelease_log(relentry) + except InvalidEntryError: + continue + releasesdata.append((relrevision, -relentry.revision, relentry, + version, release)) + releasesdata.sort() + + # collect valid releases using the versions provided by the changes and + # the packages + prevrevision = 0 + releases = [] + for (relrevision, dummy, relentry, version, release) in releasesdata: + if prevrevision == relrevision: + # ignore older markrelease of the same revision, since they + # will have no history + continue + entries = [entry for entry in currentlog + if relrevision >= entry.revision and + (prevrevision < entry.revision)] + if not entries: + #XXX probably a forced release, without commits in current/, + # check if this is the right behavior + sys.stderr.write("warning: skipping (possible) release " + "%s-%s@%s, no commits since previous markrelease (r%r)\n" % + (version, release, relrevision, prevrevision)) + continue + + release = make_release(author=relentry.author, + revision=relentry.revision, date=relentry.date, + lines=relentry.lines, entries=entries, + version=version, release=release) + releases.append(release) + prevrevision = relrevision + + # look for commits that have been not submitted (released) yet + # this is done by getting all log entries newer (greater revision no.) + # than releasesdata[-1] (in the case it exists) + if releasesdata: + latest_revision = releasesdata[-1][0] # the latest copied rev + else: + latest_revision = 0 + notsubmitted = [entry for entry in currentlog + if entry.revision > latest_revision] + if notsubmitted: + # if they are not submitted yet, what we have to do is to add + # a release/version number from getrelease() + version, release = getrelease(pkgdirurl, macros=macros, + exported=exported) + toprelease = make_release(entries=notsubmitted, released=False, + version=version, release=release) + releases.append(toprelease) + + data = dump_file(releases[::-1], currentlog=currentlog, template=template) + return data + +def _split_changelog(stream): + current = None + count = 0 + def finish(entry): + lines = entry[2] + # strip newlines at the end + for i in xrange(len(lines)-1, -1, -1): + if lines[i] != "\n": + break + del lines[i] + return entry + for line in stream: + if line.startswith("*"): + if current: + yield finish(current) + fields = line.split() + rawdate = " ".join(fields[:5]) + try: + date = time.strptime(rawdate, "* %a %b %d %Y") + except ValueError, e: + raise Error, "failed to parse spec changelog: %s" % e + curlines = [line] + current = (date, count, curlines) + # count used to ensure stable sorting when changelog entries + # have the same date, otherwise it would also compare the + # changelog lines + count -= 1 + elif current: + curlines.append(line) + else: + pass # not good, but ignore + if current: + yield finish(current) + +def sort_changelog(stream): + entries = _split_changelog(stream) + log = StringIO() + for time, count, elines in sorted(entries, reverse=True): + log.writelines(elines) + log.write("\n") + return log + +def split_spec_changelog(stream): + chlog = StringIO() + spec = StringIO() + found = 0 + visible = 0 + for line in stream: + if line.startswith("%changelog"): + found = 1 + elif not found: + spec.write(line) + elif found: + if line.strip(): + visible = 1 + chlog.write(line) + elif line.startswith("%"): + found = 0 + spec.write(line) + spec.seek(0) + if not visible: + # when there are only blanks in the changelog, make it empty + chlog = StringIO() + return spec, chlog + +def get_old_log(pkgdirurl): + chlog = StringIO() + oldurl = config.get("log", "oldurl") + if oldurl: + svn = SVN() + tmpdir = tempfile.mktemp() + try: + pkgname = layout.package_name(pkgdirurl) + pkgoldurl = os.path.join(oldurl, pkgname) + try: + # we're using HEAD here because fixes in misc/ (oldurl) may + # be newer than packages' last changed revision. + svn.export(pkgoldurl, tmpdir) + except Error: + pass + else: + logfile = os.path.join(tmpdir, "log") + if os.path.isfile(logfile): + file = open(logfile) + chlog.write("\n") # TODO needed? + log = file.read() + log = escape_macros(log) + chlog.write(log) + file.close() + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + chlog.seek(0) + return chlog + +def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, + submit=False, sort=False, template=None, macros=[], exported=None, + oldlog=False): + """Generates the changelog for a given package URL + + @another: a stream with the contents of a changelog to be merged with + the one generated + @svn: enable changelog from svn + @rev: generate the changelog with the changes up to the given + revision + @size: the number of revisions to be used (as in svn log --limit) + @submit: defines whether the latest unreleased log entries should have + the version parsed from the spec file + @sort: should changelog entries be reparsed and sorted after appending + the oldlog? + @template: the path to the cheetah template used to generate the + changelog from svn + @macros: a list of tuples containing macros to be defined when + parsing the version in the changelog + @exported: the path of a directory containing an already existing + checkout of the package, so that the spec file can be + parsed from there + @oldlog: if set it will try to append the old changelog file defined + in oldurl in mgarepo.conf + """ + newlog = StringIO() + if svn: + rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, + template=template, macros=macros, exported=exported) + newlog.write(rawsvnlog) + if another: + newlog.writelines(another) + if oldlog: + newlog.writelines(get_old_log(pkgdirurl)) + if sort: + newlog.seek(0) + newlog = sort_changelog(newlog) + newlog.seek(0) + return newlog + +def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, + submit=False, sort=False, template=None, macros=[], exported=None): + fi = open(specfile) + spec, oldchlog = split_spec_changelog(fi) + fi.close() + another = None + if config.getbool("log", "merge-spec", False): + another = oldchlog + sort = sort or config.getbool("log", "sort", False) + chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size, + submit=submit, sort=sort, template=template, macros=macros, + exported=exported, oldlog=True) + fo = open(specfile, "w") + fo.writelines(spec) + fo.write("\n\n%changelog\n") + fo.writelines(chlog) + fo.close() + +if __name__ == "__main__": + l = svn2rpm(sys.argv[1]) + print l + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/mirror.py b/MgaRepo/mirror.py new file mode 100644 index 0000000..1d1fcf3 --- /dev/null +++ b/MgaRepo/mirror.py @@ -0,0 +1,129 @@ +import sys +import os +import urlparse +import urllib + +from MgaRepo import Error, config, layout +from MgaRepo.svn import SVN + +def mirror_url(): + mirror = config.get("global", "mirror") + return mirror + +def normalize_path(url): + """normalize url for relocate_path needs""" + parsed = urlparse.urlparse(url) + path = os.path.normpath(parsed[2]) + newurl = urlparse.urlunparse((parsed[0], parsed[1], path, + parsed[3], parsed[4], parsed[5])) + return newurl + +def _joinurl(url, relpath): + parsed = urlparse.urlparse(url) + newpath = os.path.join(parsed[2], relpath) + newurl = urlparse.urlunparse((parsed[0], parsed[1], newpath, + parsed[3], parsed[4], parsed[5])) + return newurl + + +def strip_username(url): + parsed = list(urlparse.urlparse(url)) + _, parsed[1] = urllib.splituser(parsed[1]) + newurl = urlparse.urlunparse(parsed) + return newurl + +def same_base(parent, url): + """returns true if parent is parent of url""" + parent = normalize_path(parent) + url = normalize_path(url) + url = strip_username(url) + return url.startswith(parent) + +def relocate_path(oldparent, newparent, url): + oldparent = normalize_path(oldparent) + newparent = normalize_path(newparent) + url = normalize_path(url) + subpath = url[len(oldparent)+1:] + newurl = _joinurl(newparent, subpath) # subpath usually gets / at begining + return newurl + +def enabled(wcurl=None): + mirror = mirror_url() + repository = layout.repository_url() + enabled = False + if mirror and repository: + enabled = True + if wcurl and not same_base(mirror, wcurl): + enabled = False + return enabled + +def using_on(url): + """returnes True if the URL points to the mirror repository""" + mirror = mirror_url() + if mirror: + using = same_base(mirror, url) + else: + using = False + return using + +def info(url, write=False, stream=sys.stderr): + if using_on(url): + stream.write("Using the svn mirror.\n") + if write: + stream.write("To be able to commit changes, use " + "'mgarepo switch' first.\n") + +def mirror_relocate(oldparent, newparent, url, wcpath): + svn = SVN() + newurl = relocate_path(oldparent, newparent, url) + svn.switch(newurl, url, path=wcpath, relocate=True) + return newurl + +def switchto_parent(svn, url, path): + """Relocates the working copy to default_parent""" + newurl = mirror_relocate(mirror_url(), layout.repository_url(), url, path) + return newurl + +def switchto_parent_url(url): + newurl = relocate_path(mirror_url(), layout.repository_url(), url) + return newurl + +def switchto_mirror(svn, url, path): + newurl = mirror_relocate(layout.repository_url(), mirror_url(), url, path) + return newurl + +def autoswitch(svn, wcpath, wcurl, newbaseurl=None): + """Switches between mirror, default_parent, or newbaseurl""" + nobase = False + mirror = mirror_url() + repository = layout.repository_url() + current = repository + if repository is None: + raise Error, "the option repository from mgarepo.conf is "\ + "required" + indefault = same_base(repository, wcurl) + if not newbaseurl: + if not mirror: + raise Error, "an URL is needed when the option mirror "\ + "from mgarepo.conf is not set" + if indefault: + chosen = mirror + elif same_base(mirror, wcurl): + current = mirror + chosen = repository + else: + nobase = True + else: + if mirror and same_base(mirror, wcurl): + current = mirror + elif indefault: + pass # !!!! + else: + nobase = True + chosen = newbaseurl + if nobase: + raise Error, "the URL of this working copy is not based in "\ + "repository nor mirror URLs" + assert current != chosen + newurl = mirror_relocate(current, chosen, wcurl, wcpath) + return newurl diff --git a/MgaRepo/plugins/__init__.py b/MgaRepo/plugins/__init__.py new file mode 100644 index 0000000..1eeb3bb --- /dev/null +++ b/MgaRepo/plugins/__init__.py @@ -0,0 +1,27 @@ +import os + +loaded = {} + +def load(): + # based on smart's plugin system + pluginsdir = os.path.dirname(__file__) + for entry in os.listdir(pluginsdir): + if entry != "__init__.py" and entry.endswith(".py"): + name = entry[:-3] + loaded[name] = __import__("MgaRepo.plugins."+name, {}, {}, + [name]) + elif os.path.isdir(entry): + initfile = os.path.join(entry, "__init__.py") + if os.path.isfile(initfile): + loaded[entry] = __import__("MgaRepo.plugins."+entry, {}, {}, + [entry]) + +def list(): + return loaded.keys() + +def help(name): + from MgaRepo import Error + try: + return loaded[name].__doc__ + except KeyError: + raise Error, "plugin %s not found" % name diff --git a/MgaRepo/plugins/ldapusers.py b/MgaRepo/plugins/ldapusers.py new file mode 100644 index 0000000..294c32d --- /dev/null +++ b/MgaRepo/plugins/ldapusers.py @@ -0,0 +1,189 @@ +""" +A mgarepo plugin for obtaining users from a LDAP server. + +In order to enable the plugin, the user must define the following +options in the [global] section of mgarepo.conf: + + ldap-uri [required if ldap-server is unset] + the URI of the server, you can refer to more than one server by + adding more URIs separated by spaces:: + + ldap-uri = ldap://ldap.network/ ldaps://backup.network:22389/ + + ldap-server [required if ldap-uri is unset] + the host name of the LDAP server + ldap-port [optional] [default: 389] + the port of the LDAP server + ldap-base [required] + the base DN where the search will be performed + ldap-binddn [optional] [default: empty] + the DN used to bind + ldap-bindpw [optional] [default: empty] + the password used to bind + ldap-starttls [optional] [default: no] + use "yes" or "no" to enable or disable the use of the STARTTLS + LDAP extension + ldap-filterformat [optional] + [default: (&(objectClass=inetOrgPerson)(uid=$username))] + RFC-2254 filter string used in the search of the user entry. + Note that this is a python template string and will have the + user name as parameter. For example: + + ldap-filterformat = (&(objectClass=inetOrgPerson)(uid=$username)) + + Will result in the search filter: + + (&(objectClass=inetOrgPerson)(uid=john)) + + ldap-resultformat [optional] [default: $cn <$mail>] + This is a python template string. This string will be + formatted using one dict object containing the fields + returned in the LDAP search, for example: + + >>> format = Template("$cn <$mail>") + >>> d = search(basedn, filter) + >>> d + {"cn": "John Doe", "mail": "john@mandriva.org", + "uidNumber": "1290", "loginShell": "/bin/bash", + ... many other attributes ... } + >>> value = format.substitute(d) + >>> print value + John Doe <john@mandriva.org> + + Note that only the first value of the attributes will be + used. + +When the searched option is not found, it will try in mgarepo.conf. All +the values found. (including from mgarepo.conf) will be cached between +each configuration access. + +This plugin requires the package python-ldap. + +For more information, look http://qa.mandriva.com/show_bug.cgi?id=30549 +""" +from MgaRepo import Error, config + +import string + +users_cache = {} + +class LDAPError(Error): + def __init__(self, ldaperr): + self.ldaperr = ldaperr + name = ldaperr.__class__.__name__ + desc = ldaperr.message["desc"] + self.message = "LDAP error %s: %s" % (name, desc) + self.args = self.message, + +def strip_entry(entry): + "Leave only the first value in all keys in the entry" + new = dict((key, value[0]) for key, value in entry.iteritems()) + return new + +def interpolate(optname, format, data): + tmpl = string.Template(format) + try: + return tmpl.substitute(data) + except KeyError, e: + raise Error, "the key %s was not found in LDAP search, " \ + "check your %s configuration" % (e, optname) + except (TypeError, ValueError), e: + raise Error, "LDAP response formatting error: %s. Check " \ + "your %s configuration" % (e, optname) + +def used_attributes(format): + class DummyDict: + def __init__(self): + self.found = [] + def __getitem__(self, key): + self.found.append(key) + return key + dd = DummyDict() + t = string.Template(format) + t.safe_substitute(dd) + return dd.found + +def make_handler(): + uri = config.get("global", "ldap-uri") + if not uri: + server = config.get("global", "ldap-server") + if not server: + # ldap support is not enabled if ldap-uri nor ldap-server are + # defined + def dummy_wrapper(section, option=None, default=None, walk=False): + return config.get(section, option, default, wrap=False) + return dummy_wrapper + + try: + port = int(config.get("global", "ldap-port", 389)) + except ValueError: + raise Error, "the option ldap-port requires an integer, please "\ + "check your configuration files" + uri = "ldap://%s:%d" % (server, port) + + basedn = config.get("global", "ldap-base") + binddn = config.get("global", "ldap-binddn") + bindpw = config.get("global", "ldap-bindpw", "") + filterformat = config.get("global", "ldap-filterformat", + "(&(objectClass=inetOrgPerson)(uid=$username))", raw=1) + format = config.get("global", "ldap-resultformat", "$cn <$mail>", raw=1) + + valid = {"yes": True, "no": False} + raw = config.get("global", "ldap-starttls", "no") + try: + starttls = valid[raw] + except KeyError: + raise Error, "invalid value %r for ldap-starttls, use "\ + "'yes' or 'no'" % raw + + try: + import ldap + except ImportError: + raise Error, "LDAP support needs the python-ldap package "\ + "to be installed" + else: + from ldap.filter import escape_filter_chars + + def users_wrapper(section, option=None, default=None, walk=False): + global users_cache + if walk: + raise Error, "ldapusers plugin does not support user listing" + assert option is not None, \ + "When not section walking, option is required" + + value = users_cache.get(option) + if value is not None: + return value + + try: + l = ldap.initialize(uri) + if starttls: + l.start_tls_s() + if binddn: + l.bind(binddn, bindpw) + except ldap.LDAPError, e: + raise LDAPError(e) + try: + data = {"username": escape_filter_chars(option)} + filter = interpolate("ldap-filterformat", filterformat, data) + attrs = used_attributes(format) + try: + found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter, + attrlist=attrs) + except ldap.LDAPError, e: + raise LDAPError(e) + if found: + dn, entry = found[0] + entry = strip_entry(entry) + value = interpolate("ldap-resultformat", format, entry) + else: + # issue a warning? + value = config.get(section, option, default, wrap=False) + users_cache[option] = value + return value + finally: + l.unbind_s() + + return users_wrapper + +config.wrap("users", handler=make_handler()) diff --git a/MgaRepo/plugins/sample.py.txt b/MgaRepo/plugins/sample.py.txt new file mode 100644 index 0000000..846cb6d --- /dev/null +++ b/MgaRepo/plugins/sample.py.txt @@ -0,0 +1,14 @@ +# Sample mgarepo plugin. In order to test it, rename to sample.py +# vim:ft=python +from MgaRepo import config + +def users_wrapper(section, option=None, default=None, walk=False): + d = {"foolano": "Foolano De Tal <foolano@bla.com>", + "ceeclano": "Ceeclano Algumacoisa <ceeclano@bli.com>", + "beltrano": "Beltrano Bla <beltrano@mail.ru>"} + if walk: + return d.items() + + return d.get(option, default) + +config.wrap("users", handler=users_wrapper) diff --git a/MgaRepo/rpmutil.py b/MgaRepo/rpmutil.py new file mode 100644 index 0000000..2bf3ef4 --- /dev/null +++ b/MgaRepo/rpmutil.py @@ -0,0 +1,759 @@ +#!/usr/bin/python +from MgaRepo import Error, config +from MgaRepo import mirror, layout, log, binrepo +from MgaRepo.svn import SVN +from MgaRepo.simplerpm import SRPM +from MgaRepo.util import execcmd +from MgaRepo.command import default_parent +import rpm +import urlparse +import tempfile +import shutil +import string +import glob +import sys +import os + +def get_spec(pkgdirurl, targetdir=".", submit=False): + svn = SVN() + tmpdir = tempfile.mktemp() + try: + geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") + mirror.info(geturl) + svn.export("'%s'" % geturl, tmpdir) + speclist = glob.glob(os.path.join(tmpdir, "*.spec")) + if not speclist: + raise Error, "no spec files found" + spec = speclist[0] + shutil.copy(spec, targetdir) + name = os.path.basename(spec) + path = os.path.join(targetdir, name) + print "Wrote %s" % (name) + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + +def rpm_macros_defs(macros): + defs = ("--define \"%s %s\"" % macro for macro in macros) + args = " ".join(defs) + return args + +#FIXME move it to another module +def rev_touched_url(url, rev): + svn = SVN() + info = svn.info2(url) + if info is None: + raise Error, "can't fetch svn info about the URL: %s" % url + root = info["Repository Root"] + urlpath = url[len(root):] + touched = False + entries = svn.log(root, start=rev, limit=1) + entry = entries[0] + for change in entry.changed: + path = change.get("path") + if path and path.startswith(urlpath): + touched = True + return touched + +def get_srpm(pkgdirurl, + mode = "current", + targetdirs = None, + version = None, + release = None, + revision = None, + packager = "", + revname = 0, + svnlog = 0, + scripts = [], + submit = False, + template = None, + macros = [], + verbose = 0, + strict = False, + use_binrepo = False, + binrepo_check = True): + svn = SVN() + tmpdir = tempfile.mktemp() + topdir = "--define '_topdir %s'" % tmpdir + builddir = "--define '_builddir %s/%s'" % (tmpdir, "BUILD") + rpmdir = "--define '_rpmdir %s/%s'" % (tmpdir, "RPMS") + sourcedir = "--define '_sourcedir %s/%s'" % (tmpdir, "SOURCES") + specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS") + srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS") + patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES") + + try: + if mode == "version": + geturl = layout.checkout_url(pkgdirurl, version=version, + release=release) + elif mode == "pristine": + geturl = layout.checkout_url(pkgdirurl, pristine=True) + elif mode == "current" or mode == "revision": + #FIXME we should handle revisions specified using @REV + geturl = layout.checkout_url(pkgdirurl) + else: + raise Error, "unsupported get_srpm mode: %s" % mode + strict = strict or config.getbool("submit", "strict-revision", False) + if strict and not rev_touched_url(geturl, revision): + #FIXME would be nice to have the revision number even when + # revision is None + raise Error, "the revision %s does not change anything "\ + "inside %s" % (revision or "HEAD", geturl) + mirror.info(geturl) + svn.export(geturl, tmpdir, rev=revision) + if use_binrepo: + binrepo_check = (binrepo_check or + config.getbool("binrepo", "getsrpm-check", False)) + download_binaries(tmpdir, geturl, revision=revision, + export=True, check=binrepo_check) + srpmsdir = os.path.join(tmpdir, "SRPMS") + os.mkdir(srpmsdir) + specsdir = os.path.join(tmpdir, "SPECS") + speclist = glob.glob(os.path.join(specsdir, "*.spec")) + if config.getbool("srpm", "run-prep", False): + makefile = os.path.join(tmpdir, "Makefile") + if os.path.exists(makefile): + execcmd("make", "-C", tmpdir, "srpm-prep") + if not speclist: + raise Error, "no spec files found" + spec = speclist[0] + if svnlog: + submit = not not revision + log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, + template=template, macros=macros, exported=tmpdir) + for script in scripts: + #FIXME revision can be "None" + status, output = execcmd(script, tmpdir, spec, str(revision), + noerror=1) + if status != 0: + raise Error, "script %s failed" % script + if packager: + packager = " --define 'packager %s'" % packager + + defs = rpm_macros_defs(macros) + sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") + execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % + (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir, + srcrpmdir, patchdir, packager, spec, defs)) + + # copy the generated SRPMs to their target locations + targetsrpms = [] + urlrev = None + if revname: + urlrev = revision or layout.get_url_revision(geturl) + if not targetdirs: + targetdirs = (".",) + srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm")) + if not srpms: + # something fishy happened + raise Error, "no SRPMS were found at %s" % srpmsdir + for srpm in srpms: + name = os.path.basename(srpm) + if revname: + name = "@%s:%s" % (urlrev, name) + for targetdir in targetdirs: + newpath = os.path.join(targetdir, name) + targetsrpms.append(newpath) + if os.path.exists(newpath): + # should we warn? + os.unlink(newpath) + shutil.copy(srpm, newpath) + if verbose: + sys.stderr.write("Wrote: %s\n" % newpath) + return targetsrpms + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + +def patch_spec(pkgdirurl, patchfile, log=""): + #FIXME use get_spec + svn = SVN() + tmpdir = tempfile.mktemp() + try: + geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") + svn.checkout(geturl, tmpdir) + speclist = glob.glob(os.path.join(tmpdir, "*.spec")) + if not speclist: + raise Error, "no spec files found" + spec = speclist[0] + status, output = execcmd("patch", spec, patchfile) + if status != 0: + raise Error, "can't apply patch:\n%s\n" % output + else: + svn.commit(tmpdir, log="") + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + +def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, + baseurl=None, baseold=None, logmsg=None, rename=True): + svn = SVN() + srpm = SRPM(srpmfile) + tmpdir = tempfile.mktemp() + if baseurl: + pkgurl = mirror._joinurl(baseurl, srpm.name) + else: + pkgurl = layout.package_url(srpm.name, distro=branch, + mirrored=False) + print "Importing package to %s" % pkgurl + try: + if srpm.epoch: + version = "%s:%s" % (srpm.epoch, srpm.version) + else: + version = srpm.version + versionurl = "/".join([pkgurl, "releases", version]) + releaseurl = "/".join([versionurl, srpm.release]) + currenturl = "/".join([pkgurl, "current"]) + currentdir = os.path.join(tmpdir, "current") + #FIXME when pre-commit hook fails, there's no clear way to know + # what happened + ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory") + if ret or not svn.ls(currenturl, noerror=1): + svn.checkout(pkgurl, tmpdir) + svn.mkdir(os.path.join(tmpdir, "releases")) + svn.mkdir(currentdir) + svn.mkdir(os.path.join(currentdir, "SPECS")) + svn.mkdir(os.path.join(currentdir, "SOURCES")) + #svn.commit(tmpdir,log="Created package structure.") + version_exists = 1 + else: + if svn.ls(releaseurl, noerror=1): + raise Error, "release already exists" + svn.checkout("/".join([pkgurl, "current"]), tmpdir) + svn.mkdir(versionurl, noerror=1, + log="Created directory for version %s." % version) + currentdir = tmpdir + + specsdir = os.path.join(currentdir, "SPECS") + sourcesdir = os.path.join(currentdir, "SOURCES") + + unpackdir = tempfile.mktemp() + os.mkdir(unpackdir) + try: + srpm.unpack(unpackdir) + + uspecsdir = os.path.join(unpackdir, "SPECS") + usourcesdir = os.path.join(unpackdir, "SOURCES") + + uspecsentries = os.listdir(uspecsdir) + usourcesentries = os.listdir(usourcesdir) + specsentries = os.listdir(specsdir) + sourcesentries = os.listdir(sourcesdir) + + # Remove old entries + for entry in [x for x in specsentries + if x not in uspecsentries]: + if entry == ".svn": + continue + entrypath = os.path.join(specsdir, entry) + os.unlink(entrypath) + svn.remove(entrypath) + for entry in [x for x in sourcesentries + if x not in usourcesentries]: + if entry == ".svn": + continue + entrypath = os.path.join(sourcesdir, entry) + os.unlink(entrypath) + svn.remove(entrypath) + + # Copy all files + execcmd("cp -rf", uspecsdir, currentdir) + execcmd("cp -rf", usourcesdir, currentdir) + + # Add new entries + for entry in [x for x in uspecsentries + if x not in specsentries]: + entrypath = os.path.join(specsdir, entry) + svn.add(entrypath) + for entry in [x for x in usourcesentries + if x not in sourcesentries]: + entrypath = os.path.join(sourcesdir, entry) + svn.add(entrypath) + finally: + if os.path.isdir(unpackdir): + shutil.rmtree(unpackdir) + + specs = glob.glob(os.path.join(specsdir, "*.spec")) + if not specs: + raise Error, "no spec file found on %s" % specsdir + if len(specs) > 1: + raise Error, "more than one spec file found on %s" % specsdir + specpath = specs[0] + if rename: + specfile = os.path.basename(specpath) + specname = specfile[:-len(".spec")] + if specname != srpm.name: + newname = srpm.name + ".spec" + newpath = os.path.join(specsdir, newname) + sys.stderr.write("warning: renaming spec file to '%s' " + "(use -n to disable it)\n" % (newname)) + os.rename(specpath, newpath) + try: + svn.remove(specpath) + except Error: + # file not tracked + svn.revert(specpath) + svn.add(newpath) + specpath = newpath + + if striplog: + specpath = specpath + fspec = open(specpath) + spec, chlog = log.split_spec_changelog(fspec) + fspec.close() + fspec = open(specpath, "w") + fspec.writelines(spec) + fspec.close() + chlog.seek(0, os.SEEK_END) + if chlog.tell() != 0: + chlog.seek(0) + #FIXME move it to layout.py + oldurl = baseold or config.get("log", "oldurl") + pkgoldurl = mirror._joinurl(oldurl, srpm.name) + svn.mkdir(pkgoldurl, noerror=1, + log="created old log directory for %s" % srpm.name) + logtmp = tempfile.mktemp() + try: + svn.checkout(pkgoldurl, logtmp) + miscpath = os.path.join(logtmp, "log") + fmisc = open(miscpath, "w+") + fmisc.writelines(chlog) + fmisc.close() + svn.add(miscpath) + svn.commit(logtmp, + log="imported old log for %s" % srpm.name) + finally: + if os.path.isdir(logtmp): + shutil.rmtree(logtmp) + binrepo.import_binaries(currentdir, srpm.name) + svn.commit(tmpdir, + log=logmsg or ("imported package %s" % srpm.name)) + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + + # Do revision and pristine tag copies + pristineurl = layout.checkout_url(pkgurl, pristine=True) + svn.remove(pristineurl, noerror=1, + log="Removing previous pristine/ directory.") + currenturl = layout.checkout_url(pkgurl) + svn.copy(currenturl, pristineurl, + log="Copying release %s-%s to pristine/ directory." % + (version, srpm.release)) + if markrelease: + svn.copy(currenturl, releaseurl, + log="Copying release %s-%s to releases/ directory." % + (version, srpm.release)) + +def create_package(pkgdirurl, log="", verbose=0): + svn = SVN() + tmpdir = tempfile.mktemp() + try: + basename = layout.package_name(pkgdirurl) + if verbose: + print "Creating package directory...", + sys.stdout.flush() + ret = svn.mkdir(pkgdirurl, + log="Created package directory for '%s'." % basename) + if verbose: + print "done" + print "Checking it out...", + svn.checkout(pkgdirurl, tmpdir) + if verbose: + print "done" + print "Creating package structure...", + svn.mkdir(os.path.join(tmpdir, "current")) + svn.mkdir(os.path.join(tmpdir, "current", "SPECS")) + svn.mkdir(os.path.join(tmpdir, "current", "SOURCES")) + if verbose: + print "done" + print "Committing...", + svn.commit(tmpdir, + log="Created package structure for '%s'." % basename) + print "done" + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + + +def create_markrelease_log(version, release, revision): + log = """%%repsys markrelease +version: %s +release: %s +revision: %s + +%s""" % (version, release, revision, + ("Copying %s-%s to releases/ directory." % (version, release))) + return log + +def mark_release(pkgdirurl, version, release, revision): + svn = SVN() + releasesurl = layout.checkout_url(pkgdirurl, releases=True) + versionurl = "/".join([releasesurl, version]) + releaseurl = "/".join([versionurl, release]) + currenturl = layout.checkout_url(pkgdirurl) + binrepo.markrelease(currenturl, releasesurl, version, release, revision) + if svn.ls(releaseurl, noerror=1): + raise Error, "release already exists" + svn.mkdir(releasesurl, noerror=1, + log="Created releases directory.") + svn.mkdir(versionurl, noerror=1, + log="Created directory for version %s." % version) + pristineurl = layout.checkout_url(pkgdirurl, pristine=True) + svn.remove(pristineurl, noerror=1, + log="Removing previous pristine/ directory.") + svn.copy(currenturl, pristineurl, + log="Copying release %s-%s to pristine/ directory." % + (version, release)) + markreleaselog = create_markrelease_log(version, release, revision) + svn.copy(currenturl, releaseurl, rev=revision, + log=markreleaselog) + +def check_changed(pkgdirurl, all=0, show=0, verbose=0): + svn = SVN() + if all: + baseurl = pkgdirurl + packages = [] + if verbose: + print "Getting list of packages...", + sys.stdout.flush() + packages = [x[:-1] for x in svn.ls(baseurl)] + if verbose: + print "done" + if not packages: + raise Error, "couldn't get list of packages" + else: + baseurl, basename = os.path.split(pkgdirurl) + packages = [basename] + clean = [] + changed = [] + nopristine = [] + nocurrent = [] + for package in packages: + pkgdirurl = os.path.join(baseurl, package) + current = layout.checkout_url(pkgdirurl) + pristine = layout.checkout_url(pkgdirurl, pristine=True) + if verbose: + print "Checking package %s..." % package, + sys.stdout.flush() + if not svn.ls(current, noerror=1): + if verbose: + print "NO CURRENT" + nocurrent.append(package) + elif not svn.ls(pristine, noerror=1): + if verbose: + print "NO PRISTINE" + nopristine.append(package) + else: + diff = svn.diff(pristine, current) + if diff: + changed.append(package) + if verbose: + print "CHANGED" + if show: + print diff + else: + if verbose: + print "clean" + clean.append(package) + if verbose: + if not packages: + print "No packages found!" + elif all: + print "Total clean packages: %s" % len(clean) + print "Total CHANGED packages: %d" % len(changed) + print "Total NO CURRENT packages: %s" % len(nocurrent) + print "Total NO PRISTINE packages: %s" % len(nopristine) + return {"clean": clean, + "changed": changed, + "nocurrent": nocurrent, + "nopristine": nopristine} + +def checkout(pkgdirurl, path=None, revision=None, branch=None, distro=None, + spec=False, use_binrepo=False, binrepo_check=True, binrepo_link=True): + o_pkgdirurl = pkgdirurl + pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro) + append = None + if spec: + append = "SPECS" + current = layout.checkout_url(pkgdirurl, branch=branch, + append_path=append) + if path is None: + path = layout.package_name(pkgdirurl) + mirror.info(current, write=True) + svn = SVN() + svn.checkout(current, path, rev=revision, show=1) + if use_binrepo: + download_binaries(path, revision=revision, symlinks=binrepo_link, + check=binrepo_check) + +def getpkgtopdir(basedir=None): + #FIXME this implementation doesn't work well with relative path names, + # which is something we need in order to have a friendlier output + if basedir is None: + basedir = os.path.curdir + while not ispkgtopdir(basedir): + if os.path.abspath(basedir) == "/": + raise Error, "can't find top package directories SOURCES and SPECS" + basedir = os.path.join(basedir, os.path.pardir) + if basedir.startswith("./"): + basedir = basedir[2:] + return basedir + +def ispkgtopdir(path=None): + if path is None: + path = os.getcwd() + names = os.listdir(path) + return (".svn" in names and "SPECS" in names and "SOURCES" in names) + +def sync(dryrun=False, ci=False, download=False): + # TODO FIXME XXX fix it! + raise Error, "sync is not expected to work these days" + svn = SVN() + topdir = getpkgtopdir() + # run svn info because svn st does not complain when topdir is not an + # working copy + svn.info(topdir) + specsdir = os.path.join(topdir, "SPECS/") + sourcesdir = os.path.join(topdir, "SOURCES/") + for path in (specsdir, sourcesdir): + if not os.path.isdir(path): + raise Error, "%s directory not found" % path + specs = glob.glob(os.path.join(specsdir, "*.spec")) + if not specs: + raise Error, "no .spec files found in %s" % specsdir + specpath = specs[0] # FIXME better way? + try: + rpm.addMacro("_topdir", os.path.abspath(topdir)) + spec = rpm.TransactionSet().parseSpec(specpath) + except rpm.error, e: + raise Error, "could not load spec file: %s" % e + sources = dict((os.path.basename(name), name) + for name, no, flags in spec.sources()) + sourcesst = dict((os.path.basename(path), (path, st)) + for st, path in svn.status(sourcesdir, noignore=True)) + toadd_br = [] + toadd_svn = [] + toremove_svn = [] + toremove_br = [] + # add the spec file itself, in case of a new package + specstl = svn.status(specpath, noignore=True) + if specstl: + specst, _ = specstl[0] + if specst == "?": + toadd_svn.append(specpath) + # add source files: + for source, url in sources.iteritems(): + sourcepath = os.path.join(sourcesdir, source) + if sourcesst.get(source): + if not os.path.islink(sourcepath): + if not binrepo.is_tracked(sourcepath): + if binrepo.is_binary(sourcepath): + toadd_br.append(sourcepath) + else: + toadd_svn.append(sourcepath) + else: + sys.stderr.write("warning: %s not found\n" % sourcepath) + elif download and not os.path.isfile(sourcepath): + print "%s not found, downloading from %s" % (sourcepath, url) + fmt = config.get("global", "download-command", + "wget -c -O '$dest' $url") + context = {"dest": sourcepath, "url": url} + try: + cmd = string.Template(fmt).substitute(context) + except KeyError, e: + raise Error, "invalid variable %r in download-command "\ + "configuration option" % e + execcmd(cmd, show=True) + if os.path.isfile(sourcepath): + if binrepo.is_binary(sourcepath): + toadd_br.append(sourcepath) + else: + toadd_svn.append(sourcepath) + else: + raise Error, "file not found: %s" % sourcepath + # rm entries not found in sources and still in svn + found = os.listdir(sourcesdir) + for entry in found: + if entry == ".svn" or entry == "sources": + continue + status = sourcesst.get(entry) + path = os.path.join(sourcesdir, entry) + if entry not in sources: + if status is None: # file is tracked by svn + toremove_svn.append(path) + elif binrepo.is_tracked(path): + toremove_br.append(path) + for path in toremove_svn: + print "D\t%s" % path + if not dryrun: + svn.remove(path, local=True) + for path in toremove_br: + print "DB\t%s" % path + if not dryrun: + binrepo.delete_pending(path) + for path in toadd_svn: + print "A\t%s" % path + if not dryrun: + svn.add(path, local=True) + for path in toadd_br: + print "AB\t%s" % path + if not dryrun: + binrepo.upload_pending(path) + if commit: + commit(topdir) + +def commit(target=".", message=None, logfile=None): + topdir = getpkgtopdir(target) + sourcesdir = os.path.join(topdir, "SOURCES") + binrepo.commit(sourcesdir) #TODO make it optional + svn = SVN() + status = svn.status(target, quiet=True) + if not status: + print "nothing to commit" + return + info = svn.info2(target) + url = info.get("URL") + if url is None: + raise Error, "working copy URL not provided by svn info" + mirrored = mirror.using_on(url) + if mirrored: + newurl = mirror.switchto_parent(svn, url, target) + print "relocated to", newurl + # we can't use the svn object here because svn --non-interactive option + # hides VISUAL + opts = [] + if message is not None: + opts.append("-m \"%s\"" % message) + if logfile is not None: + opts.append("-F \"%s\"" % logfile) + mopts = " ".join(opts) + os.system("svn ci %s %s" % (mopts, target)) + if mirrored: + print "use \"mgarepo switch\" in order to switch back to mirror "\ + "later" + +def spec_sources(topdir): + specs = glob.glob(os.path.join(topdir, "SPECS/*.spec")) + spec_path = specs[0] # FIXME use svn info to ensure which one + ts = rpm.ts() + spec = ts.parseSpec(spec_path) + sources = [name for name, x, y in spec.sources()] + return sources + +def download_binaries(target, pkgdirurl=None, export=False, revision=None, + symlinks=True, check=False): + refurl = pkgdirurl + if refurl is None: + refurl = binrepo.svn_root(target) + if binrepo.enabled(refurl): + binrepo.download(target, pkgdirurl, export=export, + revision=revision, symlinks=symlinks, check=check) + +def update(target=None): + svn = SVN() + info = None + svn_target = None + br_target = None + if target: + svn_target = target + else: + top = getpkgtopdir() + svn_target = top + br_target = top + if svn_target: + svn.update(svn_target, show=True) + if br_target: + info = svn.info2(svn_target) + if not br_target and not svn_target: + raise Error, "target not in SVN nor in binaries "\ + "repository: %s" % target + url = info["URL"] + download_binaries(br_target, url) + +def upload(paths): + for path in paths: + binrepo.upload(path) + +def binrepo_delete(paths, commit=False): + #TODO handle files tracked by svn + refurl = binrepo.svn_root(paths[0]) + if not binrepo.enabled(refurl): + raise Error, "binary repository is not enabled for %s" % refurl + added, deleted = binrepo.remove(paths) + if commit: + svn = SVN() + spath = binrepo.sources_path(paths[0]) + log = _sources_log(added, deleted) + svn.commit(spath, log=log) + +def switch(mirrorurl=None): + svn = SVN() + topdir = getpkgtopdir() + info = svn.info2(topdir) + wcurl = info.get("URL") + if wcurl is None: + raise Error, "working copy URL not provided by svn info" + newurl = mirror.autoswitch(svn, topdir, wcurl, mirrorurl) + print "switched to", newurl + +def get_submit_info(path): + path = os.path.abspath(path) + + # First, look for SPECS and SOURCES directories. + found = False + while path != "/": + if os.path.isdir(path): + specsdir = os.path.join(path, "SPECS") + sourcesdir = os.path.join(path, "SOURCES") + if os.path.isdir(specsdir) and os.path.isdir(sourcesdir): + found = True + break + path = os.path.dirname(path) + if not found: + raise Error, "SPECS and/or SOURCES directories not found" + + # Then, check if this is really a subversion directory. + if not os.path.isdir(os.path.join(path, ".svn")): + raise Error, "subversion directory not found" + + svn = SVN() + + # Now, extract the package name. + info = svn.info2(path) + url = info.get("URL") + if url is None: + raise Error, "missing URL from svn info %s" % path + toks = url.split("/") + if len(toks) < 2 or toks[-1] != "current": + raise Error, "unexpected URL received from 'svn info'" + name = toks[-2] + url = "/".join(toks[:-1]) + + # Finally, guess revision. + max = -1 + files = [] + files.extend(glob.glob("%s/*" % specsdir)) + files.extend(glob.glob("%s/*" % sourcesdir)) + for file in files: + try: + info = svn.info2(file) + except Error: + # possibly not tracked + continue + if info is None: + continue + rawrev = info.get("Last Changed Rev") + if rawrev: + rev = int(rawrev) + if rev > max: + max = rev + if max == -1: + raise Error, "revision tag not found in 'svn info' output" + + if mirror.using_on(url): + url = mirror.switchto_parent_url(url) + + return name, url, max + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/simplerpm.py b/MgaRepo/simplerpm.py new file mode 100644 index 0000000..806079c --- /dev/null +++ b/MgaRepo/simplerpm.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +from MgaRepo.util import execcmd + +class SRPM: + def __init__(self, filename): + self.filename = filename + self._getinfo() + + def _getinfo(self): + cmdstr = "rpm -qp --qf '%%{name} %%{epoch} %%{release} %%{version}' %s" + status, output = execcmd(cmdstr % self.filename) + self.name, self.epoch, self.release, self.version = output.split() + if self.epoch == "(none)": + self.epoch = None + + def unpack(self, topdir): + execcmd("rpm -i --define '_topdir %s' %s" % (topdir, self.filename)) + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/svn.py b/MgaRepo/svn.py new file mode 100644 index 0000000..d44d7f2 --- /dev/null +++ b/MgaRepo/svn.py @@ -0,0 +1,430 @@ +from MgaRepo import Error, SilentError, config +from MgaRepo.util import execcmd, get_auth +import sys +import os +import re +import time + +__all__ = ["SVN", "SVNLook", "SVNLogEntry"] + +class SVNLogEntry: + def __init__(self, revision, author, date): + self.revision = revision + self.author = author + self.date = date + self.changed = [] + self.lines = [] + + def __cmp__(self, other): + return cmp(self.date, other.date) + +class SVN: + def _execsvn(self, *args, **kwargs): + localcmds = ("add", "revert", "cleanup") + if not kwargs.get("show") and args[0] not in localcmds: + args = list(args) + args.append("--non-interactive") + else: + kwargs["geterr"] = True + kwargs["cleanerr"] = True + if kwargs.get("xml"): + args.append("--xml") + self._set_env() + svn_command = config.get("global", "svn-command", "svn") + cmdstr = svn_command + " " + " ".join(args) + try: + return execcmd(cmdstr, **kwargs) + except Error, e: + msg = None + if e.args: + if "Permission denied" in e.args[0]: + msg = ("It seems ssh-agent or ForwardAgent are not setup " + "or your username is wrong. See " + "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" + " for more information.") + elif "authorization failed" in e.args[0]: + msg = ("Note that mgarepo does not support any HTTP " + "authenticated access.") + if kwargs.get("show") and \ + not config.getbool("global", "verbose", 0): + # svn has already dumped error messages, we don't need to + # do it too + if msg: + sys.stderr.write("\n") + sys.stderr.write(msg) + sys.stderr.write("\n") + raise SilentError + elif msg: + raise Error, "%s\n%s" % (e, msg) + raise + + def _set_env(self): + wrapper = "mgarepo-ssh" + repsys = config.get("global", "mgarepo-cmd") + if repsys: + dir = os.path.dirname(repsys) + path = os.path.join(dir, wrapper) + if os.path.exists(path): + wrapper = path + defaults = {"SVN_SSH": wrapper} + os.environ.update(defaults) + raw = config.get("global", "svn-env") + if raw: + for line in raw.split("\n"): + env = line.strip() + if not env: + continue + try: + name, value = env.split("=", 1) + except ValueError: + sys.stderr.write("invalid svn environment line: %r\n" % env) + continue + os.environ[name] = value + + def _execsvn_success(self, *args, **kwargs): + status, output = self._execsvn(*args, **kwargs) + return status == 0 + + def _add_log(self, cmd_args, received_kwargs, optional=0): + if (not optional or + received_kwargs.has_key("log") or + received_kwargs.has_key("logfile")): + ret = received_kwargs.get("log") + if ret is not None: + cmd_args.append("-m '%s'" % ret) + ret = received_kwargs.get("logfile") + if ret is not None: + cmd_args.append("-F '%s'" % ret) + + def _add_revision(self, cmd_args, received_kwargs, optional=0): + if not optional or received_kwargs.has_key("rev"): + ret = received_kwargs.get("rev") + if isinstance(ret, basestring): + if not ret.startswith("{"): # if not a datespec + try: + ret = int(ret) + except ValueError: + raise Error, "invalid revision provided" + if ret: + cmd_args.append("-r '%s'" % ret) + + def add(self, path, **kwargs): + cmd = ["add", path + '@'] + return self._execsvn_success(noauth=1, *cmd, **kwargs) + + def copy(self, pathfrom, pathto, **kwargs): + cmd = ["copy", pathfrom + '@', pathto + '@'] + self._add_revision(cmd, kwargs, optional=1) + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def remove(self, path, force=0, **kwargs): + cmd = ["remove", path + '@'] + self._add_log(cmd, kwargs) + if force: + cmd.append("--force") + return self._execsvn_success(*cmd, **kwargs) + + def mkdir(self, path, **kwargs): + cmd = ["mkdir", path + '@'] + if kwargs.get("parents"): + cmd.append("--parents") + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def _execsvn_commit(self, *cmd, **kwargs): + status, output = self._execsvn(*cmd, **kwargs) + match = re.search("Committed revision (?P<rev>\\d+)\\.$", output) + if match: + rawrev = match.group("rev") + return int(rawrev) + + def commit(self, path, **kwargs): + cmd = ["commit", path + '@'] + if kwargs.get("nonrecursive"): + cmd.append("-N") + self._add_log(cmd, kwargs) + return self._execsvn_commit(*cmd, **kwargs) + + def import_(self, path, url, **kwargs): + cmd = ["import", "'%s'" % path, "'%s'" % url] + self._add_log(cmd, kwargs) + return self._execsvn_commit(*cmd, **kwargs) + + def export(self, url, targetpath, **kwargs): + cmd = ["export", "'%s'" % url, targetpath] + self._add_revision(cmd, kwargs, optional=1) + return self._execsvn_success(*cmd, **kwargs) + + def checkout(self, url, targetpath, **kwargs): + cmd = ["checkout", "'%s'" % url, targetpath] + self._add_revision(cmd, kwargs, optional=1) + return self._execsvn_success(*cmd, **kwargs) + + def propget(self, propname, targets, **kwargs): + cmd = ["propget", propname, targets] + if kwargs.get("revprop"): + cmd.append("--revprop") + self._add_revision(cmd, kwargs) + status, output = self._execsvn(local=True, *cmd, **kwargs) + return output + + def propset(self, propname, value, targets, **kwargs): + cmd = ["propset", propname, "'%s'" % value, targets] + return self._execsvn_success(*cmd, **kwargs) + + def propedit(self, propname, target, **kwargs): + cmd = ["propedit", propname, target] + if kwargs.get("rev"): + cmd.append("--revprop") + self._add_revision(cmd, kwargs) + return self._execsvn_success(local=True, show=True, *cmd, **kwargs) + + def revision(self, path, **kwargs): + cmd = ["info", path + '@'] + status, output = self._execsvn(local=True, *cmd, **kwargs) + if status == 0: + for line in output.splitlines(): + if line.startswith("Last Changed Rev: "): + return int(line.split()[3]) + return None + + def info(self, path, **kwargs): + cmd = ["info", path + '@'] + status, output = self._execsvn(local=True, noerror=True, *cmd, **kwargs) + if "Not a versioned resource" not in output: + return output.splitlines() + return None + + def info2(self, *args, **kwargs): + lines = self.info(*args, **kwargs) + if lines is None: + return None + pairs = [[w.strip() for w in line.split(":", 1)] for line in lines] + info = dict(pairs) + return info + + def ls(self, path, **kwargs): + cmd = ["ls", path + '@'] + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output.split() + return None + + def status(self, path, **kwargs): + cmd = ["status", path + '@'] + if kwargs.get("verbose"): + cmd.append("-v") + if kwargs.get("noignore"): + cmd.append("--no-ignore") + if kwargs.get("quiet"): + cmd.append("--quiet") + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.splitlines()] + return None + + def cleanup(self, path, **kwargs): + cmd = ["cleanup", path + '@'] + return self._execsvn_success(*cmd, **kwargs) + + def revert(self, path, **kwargs): + cmd = ["revert", path + '@'] + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def switch(self, url, oldurl=None, path=None, relocate=False, **kwargs): + cmd = ["switch"] + if relocate: + if oldurl is None: + raise Error, "You must supply the old URL when "\ + "relocating working copies" + cmd.append("--relocate") + cmd.append(oldurl) + cmd.append(url) + if path is not None: + cmd.append(path) + return self._execsvn_success(*cmd, **kwargs) + + def update(self, path, **kwargs): + cmd = ["update", path + '@'] + self._add_revision(cmd, kwargs, optional=1) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def merge(self, url1, url2=None, rev1=None, rev2=None, path=None, + **kwargs): + cmd = ["merge"] + if rev1 and rev2 and not url2: + cmd.append("-r") + cmd.append("%s:%s" % (rev1, rev2)) + cmd.append(url1) + else: + if not url2: + raise ValueError, \ + "url2 needed if two revisions are not provided" + if rev1: + cmd.append("%s@%s" % (url1, rev1)) + else: + cmd.append(url1) + if rev2: + cmd.append("%s@%s" % (url2, rev2)) + else: + cmd.append(url2) + if path: + cmd.append(path) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def diff(self, pathurl1, pathurl2=None, **kwargs): + cmd = ["diff", pathurl1] + self._add_revision(cmd, kwargs, optional=1) + if pathurl2: + cmd.append(pathurl2) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output + return None + + def cat(self, url, **kwargs): + cmd = ["cat", url] + self._add_revision(cmd, kwargs, optional=1) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output + return None + + def log(self, url, start=None, end=0, limit=None, **kwargs): + cmd = ["log", "-v", url] + if start is not None or end != 0: + if start is not None and type(start) is not type(0): + try: + start = int(start) + except (ValueError, TypeError): + raise Error, "invalid log start revision provided" + if type(end) is not type(0): + try: + end = int(end) + except (ValueError, TypeError): + raise Error, "invalid log end revision provided" + start = start or "HEAD" + cmd.append("-r %s:%s" % (start, end)) + if limit is not None: + try: + limit = int(limit) + except (ValueError, TypeError): + raise Error, "invalid limit number provided" + cmd.append("--limit %d" % limit) + status, output = self._execsvn(*cmd, **kwargs) + if status != 0: + return None + + revheader = re.compile("^r(?P<revision>[0-9]+) \| (?P<author>[^\|]+) \| (?P<date>[^\|]+) \| (?P<lines>[0-9]+) (?:line|lines)$") + changedpat = re.compile(r"^\s+(?P<action>[^\s]+) (?P<path>[^\s]+)(?: \([^\s]+ (?P<from_path>[^:]+)(?:\:(?P<from_rev>[0-9]+))?\))?$") + logseparator = "-"*72 + linesleft = 0 + entry = None + log = [] + appendchanged = 0 + changedheader = 0 + for line in output.splitlines(): + line = line.rstrip() + if changedheader: + appendchanged = 1 + changedheader = 0 + elif appendchanged: + if not line: + appendchanged = 0 + continue + m = changedpat.match(line) + if m: + changed = m.groupdict().copy() + from_rev = changed.get("from_rev") + if from_rev is not None: + try: + changed["from_rev"] = int(from_rev) + except (ValueError, TypeError): + raise Error, "invalid revision number in svn log" + entry.changed.append(changed) + elif linesleft == 0: + if line != logseparator: + m = revheader.match(line) + if m: + linesleft = int(m.group("lines")) + timestr = " ".join(m.group("date").split()[:2]) + timetuple = time.strptime(timestr, + "%Y-%m-%d %H:%M:%S") + entry = SVNLogEntry(int(m.group("revision")), + m.group("author"), timetuple) + log.append(entry) + changedheader = 1 + else: + entry.lines.append(line) + linesleft -= 1 + log.sort() + log.reverse() + return log + +class SVNLook: + def __init__(self, repospath, txn=None, rev=None): + self.repospath = repospath + self.txn = txn + self.rev = rev + + def _execsvnlook(self, cmd, *args, **kwargs): + execcmd_args = ["svnlook", cmd, self.repospath] + self._add_txnrev(execcmd_args, kwargs) + execcmd_args += args + execcmd_kwargs = {} + keywords = ["show", "noerror"] + for key in keywords: + if kwargs.has_key(key): + execcmd_kwargs[key] = kwargs[key] + return execcmd(*execcmd_args, **execcmd_kwargs) + + def _add_txnrev(self, cmd_args, received_kwargs): + if received_kwargs.has_key("txn"): + txn = received_kwargs.get("txn") + if txn is not None: + cmd_args += ["-t", txn] + elif self.txn is not None: + cmd_args += ["-t", self.txn] + if received_kwargs.has_key("rev"): + rev = received_kwargs.get("rev") + if rev is not None: + cmd_args += ["-r", rev] + elif self.rev is not None: + cmd_args += ["-r", self.rev] + + def changed(self, **kwargs): + status, output = self._execsvnlook("changed", **kwargs) + if status != 0: + return None + changes = [] + for line in output.splitlines(): + line = line.rstrip() + if not line: + continue + entry = [None, None, None] + changedata, changeprop, path = None, None, None + if line[0] != "_": + changedata = line[0] + if line[1] != " ": + changeprop = line[1] + path = line[4:] + changes.append((changedata, changeprop, path)) + return changes + + def author(self, **kwargs): + status, output = self._execsvnlook("author", **kwargs) + if status != 0: + return None + return output.strip() + +# vim:et:ts=4:sw=4 diff --git a/MgaRepo/util.py b/MgaRepo/util.py new file mode 100644 index 0000000..69f3a79 --- /dev/null +++ b/MgaRepo/util.py @@ -0,0 +1,141 @@ +#!/usr/bin/python + +from MgaRepo import Error, config + +import subprocess +import getpass +import sys +import os +import re +import logging +from cStringIO import StringIO +#import commands + +log = logging.getLogger("mgarepo") + +# Our own version of commands' getstatusoutput(). We have a commands +# module directory, so we can't import Python's standard module +def commands_getstatusoutput(cmd): + """Return (status, output) of executing cmd in a shell.""" + import os + pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r') + text = pipe.read() + sts = pipe.close() + if sts is None: sts = 0 + if text[-1:] == '\n': text = text[:-1] + return sts, text + +def execcmd(*cmd, **kwargs): + cmdstr = " ".join(cmd) + if kwargs.get("show"): + if kwargs.get("geterr"): + err = StringIO() + pipe = subprocess.Popen(cmdstr, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + of = pipe.stdout.fileno() + ef = pipe.stderr.fileno() + while True: + odata = os.read(of, 8192) + sys.stdout.write(odata) + edata = os.read(ef, 8192) + err.write(edata) + sys.stderr.write(edata) + status = pipe.poll() + if status is not None and not (odata and edata): + break + output = err.getvalue() + else: + status = os.system(cmdstr) + output = "" + else: + status, output = commands_getstatusoutput( + "LANG=C LANGUAGE=C LC_ALL=C "+cmdstr) + verbose = config.getbool("global", "verbose", 0) + if status != 0 and not kwargs.get("noerror"): + if kwargs.get("cleanerr") and not verbose: + raise Error, output + else: + raise Error, "command failed: %s\n%s\n" % (cmdstr, output) + if verbose: + print cmdstr + sys.stdout.write(output) + return status, output + +def get_auth(username=None, password=None): + set_username = 1 + set_password = 1 + if not username: + username = config.get("auth", "username") + if not username: + username = raw_input("username: ") + else: + set_username = 0 + if not password: + password = config.get("auth", "password") + if not password: + password = getpass.getpass("password: ") + else: + set_password = 0 + if set_username: + config.set("auth", "username", username) + if set_password: + config.set("auth", "password", password) + return username, password + + +def mapurl(url): + """Maps a url following the regexp provided by the option url-map in + mgarepo.conf + """ + urlmap = config.get("global", "url-map") + newurl = url + if urlmap: + try: + expr_, replace = urlmap.split()[:2] + except ValueError: + log.error("invalid url-map: %s", urlmap) + else: + try: + newurl = re.sub(expr_, replace, url) + except re.error, errmsg: + log.error("error in URL mapping regexp: %s", errmsg) + return newurl + + +def get_helper(name): + """Tries to find the path of a helper script + + It first looks if the helper has been explicitly defined in + configuration, if not, falls back to the default helper path, which can + also be defined in configuration file(s). + """ + helperdir = config.get("helper", "prefix", "/usr/share/mgarepo") + hpath = config.get("helper", name, None) or \ + os.path.join(helperdir, name) + if not os.path.isfile(hpath): + log.warn("providing unexistent helper: %s", hpath) + return hpath + +def rellink(src, dst): + """Creates relative symlinks + + It will find the common ancestor and append to the src path. + """ + asrc = os.path.abspath(src) + adst = os.path.abspath(dst) + csrc = asrc.split(os.path.sep) + cdst = adst.split(os.path.sep) + dstname = cdst.pop() + i = 0 + l = min(len(csrc), len(cdst)) + while i < l: + if csrc[i] != cdst[i]: + break + i += 1 + dstextra = len(cdst[i:]) + steps = [os.path.pardir] * dstextra + steps.extend(csrc[i:]) + return os.path.sep.join(steps) + + +# vim:et:ts=4:sw=4 |