From b2ce8ef1c40f7c58f4bb4629b5b5e95ce8c252d2 Mon Sep 17 00:00:00 2001 From: Bogdano Arendartchuk Date: Mon, 4 Jun 2007 15:03:57 +0000 Subject: Frontported changes from V1_6_X since april --- CHANGES | 33 +++++ MANIFEST.in | 6 +- Makefile | 1 + README.LDAP | 52 ++++++++ RepSys/ConfigParser.py | 45 ++++++- RepSys/__init__.py | 21 ++++ RepSys/cgiutil.py | 24 +++- RepSys/commands/changed.py | 2 +- RepSys/commands/ci.py | 29 +++++ RepSys/commands/co.py | 2 +- RepSys/commands/editlog.py | 3 +- RepSys/commands/markrelease.py | 2 +- RepSys/commands/submit.py | 85 +++++-------- RepSys/commands/sync.py | 31 +++++ RepSys/log.py | 276 +++++++++++++++++++++++++++++------------ RepSys/mirror.py | 42 +++++++ RepSys/plugins/__init__.py | 27 ++++ RepSys/plugins/ldapusers.py | 164 ++++++++++++++++++++++++ RepSys/plugins/sample.py.txt | 14 +++ RepSys/rpmutil.py | 141 +++++++++++++++++---- RepSys/svn.py | 93 +++++++++++--- TODO.LDAP | 4 + create-srpm | 20 ++- default.chlog | 14 ++- repsys | 34 ++++- repsys.conf | 33 +++-- 26 files changed, 996 insertions(+), 202 deletions(-) create mode 100644 README.LDAP create mode 100644 RepSys/commands/ci.py create mode 100644 RepSys/commands/sync.py create mode 100644 RepSys/mirror.py create mode 100644 RepSys/plugins/__init__.py create mode 100644 RepSys/plugins/ldapusers.py create mode 100644 RepSys/plugins/sample.py.txt create mode 100644 TODO.LDAP diff --git a/CHANGES b/CHANGES index aa9e004..ce1fa7e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,36 @@ +* 1.6.18 +- initialize plugins in create-srpm too + +* 1.6.17 +- brought from mdvsys world the sync command +- ldapusers: the configuration format has changed, now it uses python + template strings +- ldapusers: many fixes: better error messages, ldap-port working, results + contain only the fields needed, unbinding after search, filters are + escaped + +* 1.6.16 +- introduced the plugin ldapusers: repsys user data obtained from LDAP; + this plugin is builtin +- added support to plugins, and the hability to wrap configuration sections +- added workaround in the template to ignore empty releases +- added initial support to mirrors, as requested by mrl; it required the + new subcommand "ci" +- changelogs from misc/ will now should come from HEAD and should be + escaped (%%) + +* 1.6.15 +- empty changelog entries are now shown, with a EMPTYLOG tag to allow + rpmlint warn the developer about it +- check (and warn) if a temporary package has already been removed before + trying to remove it + +* 1.6.2b +- make submit pass --define options to create-srpm script +- print error message when create-srpm fails +- make get_srpm return the srpms list +- add upload-srpm support in create-srpm + * 1.6.2a - moved revision-offset to [log] section and added a comment diff --git a/MANIFEST.in b/MANIFEST.in index 9311d09..1264d01 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,6 @@ recursive-include RepSys *.py -include repsys repsys.conf MANIFEST.in +include RepSys/plugins/*.txt +include repsys repsys.conf MANIFEST.in +include README.LDAP +include *.chlog +include rebrand-mdk create-srpm getsrpm-mdk diff --git a/Makefile b/Makefile index 46ddead..e1aa91b 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ RELEASE:=$(shell rpm -q --qf %{RELEASE} --specfile $(PACKAGE).spec) TAG := $(shell echo "V$(VERSION)_$(RELEASE)" | tr -- '-.' '__') FILES = ChangeLog Makefile MANIFEST.in PKG-INFO create-srpm getsrpm-mdk rebrand-mdk \ + {compatv15,default,oldfashion,revno}.chlog \ repsys repsys.conf repsys.spec setup.cfg setup.py RepSys/*.py RepSys/{cgi,commands}/*.py # rules to build a test rpm diff --git a/README.LDAP b/README.LDAP new file mode 100644 index 0000000..863be6d --- /dev/null +++ b/README.LDAP @@ -0,0 +1,52 @@ +A Repsys plugin for obtaining users from a LDAP server. + +In order to enable the plugin, the user must define the following +options in the [global] section of repsys.conf: + + ldap-server [required] + the host name of the LDAP server + ldap-port [optional] [default: 389] + the port of the LDAP server + ldap-base [required] + the base DN where the search will be performed + ldap-binddn [optional] [default: empty] + the DN used to bind + ldap-bindpw [optional] [default: empty] + the password used to bind + ldap-filterformat [optional] + [default: (&(objectClass=inetOrgPerson)(uid=$username))] + RFC-2254 filter string used in the search of the user entry. + Note that this is a python template string and will have the + user name as parameter. For example: + + ldap-filterformat = (&(objectClass=inetOrgPerson)(uid=$username)) + + Will result in the search filter: + + (&(objectClass=inetOrgPerson)(uid=john)) + + ldap-resultformat [optional] [default: $cn <$mail>] + This is a python template string. This string will be + formatted using one dict object containing the fields + returned in the LDAP search, for example: + + >>> format = Template("$cn <$mail>") + >>> d = search(basedn, filter) + >>> d + {"cn": "John Doe", "mail": "john@mandriva.org", + "uidNumber": "1290", "loginShell": "/bin/bash", + ... many other attributes ... } + >>> value = format.substitute(d) + >>> print value + John Doe + + Note that only the first value of the attributes will be + used. + +When the searched option is not found, it will try in repsys.conf. All +the values found. (including from repsys.conf) will be cached between +each configuration access. + +This plugin requires the package python-ldap. + +For more information, look http://qa.mandriva.com/show_bug.cgi?id=30549 diff --git a/RepSys/ConfigParser.py b/RepSys/ConfigParser.py index 0f219b9..4dc3e3c 100644 --- a/RepSys/ConfigParser.py +++ b/RepSys/ConfigParser.py @@ -348,6 +348,7 @@ import os class Config: def __init__(self): self._config = ConfigParser() + self._wrapped = {} conffiles = [] conffiles.append("/etc/repsys.conf") repsys_conf = os.environ.get("REPSYS_CONF") @@ -358,6 +359,14 @@ class Config: if os.path.isfile(file): self._config.read(file) + def wrap(self, section, handler, option=None): + """Set one wrapper for a given section + + The wrapper must be a function + f(section, option=None, default=None, walk=False). + """ + self._wrapped[section] = handler + def sections(self): try: return self._config.sections() @@ -373,12 +382,20 @@ class Config: def set(self, section, option, value): return self._config.set(section, option, value) - def walk(self, section): - return self._config.walk(section) - - def get(self, section, option, default=None): + def walk(self, section, option=None, raw=0, vars=None): + handler = self._wrapped.get(section) + if handler: + return handler(section, option, walk=True) + return self._config.walk(section, option, raw, vars) + + def get(self, section, option, default=None, raw=False, wrap=True): + if wrap: + handler = self._wrapped.get(section) + if handler: + handler = self._wrapped.get(section) + return handler(section, option, default) try: - return self._config.get(section, option) + return self._config.get(section, option, raw=raw) except Error: return default @@ -395,4 +412,22 @@ class Config: return states[ret.lower()] return default +def test(): + config = Config() + def handler(section, option=None, default=None, walk=False): + d = {"fulano": "ciclano", + "foolano": "ceeclano"} + if walk: + return d.items() + elif option in d: + return d[option] + else: + return config.get(section, option, default, wrap=False) + config.wrap("users", handler=handler) + print config.get("users", "fulano") # found in wrapper + print config.get("users", "andreas") # found in repsys.conf + print config.walk("users") + +if __name__ == "__main__": + test() # vim:ts=4:sw=4:et diff --git a/RepSys/__init__.py b/RepSys/__init__.py index c371e7a..b303065 100644 --- a/RepSys/__init__.py +++ b/RepSys/__init__.py @@ -1,9 +1,30 @@ #!/usr/bin/python +import re +import os +import tempfile import ConfigParser + config = ConfigParser.Config() +tempfile.tempdir = config.get("global", "tempdir", None) or None # when "" del ConfigParser class Error(Exception): pass +class RepSysTree: + """ + This class just hold methods that abstract all the not-so-explicit + rules about the directory structure of a repsys repository. + """ + def fixpath(cls, url): + return re.sub("/+$", "", url) + fixpath = classmethod(fixpath) + + def pkgname(cls, pkgdirurl): + # we must remove trailling slashes in the package path because + # os.path.basename could return "" from URLs ending with "/" + fixedurl = cls.fixpath(pkgdirurl) + return os.path.basename(fixedurl) + pkgname = classmethod(pkgname) + # vim:et:ts=4:sw=4 diff --git a/RepSys/cgiutil.py b/RepSys/cgiutil.py index 6dda91e..35c5efb 100644 --- a/RepSys/cgiutil.py +++ b/RepSys/cgiutil.py @@ -1,6 +1,7 @@ #!/usr/bin/python from RepSys import Error, config from RepSys.svn import SVN +from RepSys.ConfigParser import NoSectionError import time import re @@ -10,11 +11,23 @@ class SubmitTarget: def __init__(self): self.name = "" self.target = "" + self.macros = [] self.allowed = [] self.scripts = [] TARGETS = [] +def parse_macrosref(refs, config): + macros = [] + for name in refs: + secname = "macros %s" % name + try: + macros.extend(config.walk(secname, raw=True)) + except NoSectionError: + raise Error, "missing macros section " \ + "%r in configuration" % secname + return macros + def get_targets(): global TARGETS if not TARGETS: @@ -27,12 +40,11 @@ def get_targets(): target = SubmitTarget() target.name = m.group(1) for option, value in config.walk(section): - if option == "target": - target.target = value.split() - elif option == "allowed": - target.allowed = value.split() - elif option == "scripts": - target.scripts = value.split() + if option in ("target", "allowed", "scripts"): + setattr(target, option, value.split()) + elif option == "rpm-macros": + refs = value.split() + target.macros = parse_macrosref(refs, config) else: raise Error, "unknown [%s] option %s" % (section, option) TARGETS.append(target) diff --git a/RepSys/commands/changed.py b/RepSys/commands/changed.py index c99f3ae..d3094a8 100644 --- a/RepSys/commands/changed.py +++ b/RepSys/commands/changed.py @@ -25,7 +25,7 @@ def parse_options(): opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" - opts.url = default_parent(args[0]) + opts.pkgdirurl = default_parent(args[0]) opts.verbose = 1 # Unconfigurable return opts diff --git a/RepSys/commands/ci.py b/RepSys/commands/ci.py new file mode 100644 index 0000000..9ffa3bd --- /dev/null +++ b/RepSys/commands/ci.py @@ -0,0 +1,29 @@ +#!/usr/bin/python +from RepSys.command import * +from RepSys.rpmutil import commit + +HELP = """\ +Usage: repsys ci [TARGET] + +Will commit a change. The difference between an ordinary "svn ci" and +"repsys ci" is that it relocates the working copy to the default repository +in case the option "mirror" is set in repsys.conf. + +Options: + -h Show this message + +Examples: + repsys ci + repsys ci SPECS/package.spec SPECS/package-patch.patch +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-m", dest="message", default=None) + opts, args = parser.parse_args() + if len(args): + opts.target = args[0] + return opts + +def main(): + do_command(parse_options, commit) diff --git a/RepSys/commands/co.py b/RepSys/commands/co.py index 0c9d2dc..f2b4d64 100644 --- a/RepSys/commands/co.py +++ b/RepSys/commands/co.py @@ -23,7 +23,7 @@ def parse_options(): opts, args = parser.parse_args() if len(args) not in (1, 2): raise Error, "invalid arguments" - opts.url = default_parent(args[0]) + opts.pkgdirurl = default_parent(args[0]) if len(args) == 2: opts.path = args[1] else: diff --git a/RepSys/commands/editlog.py b/RepSys/commands/editlog.py index a98b761..367238f 100644 --- a/RepSys/commands/editlog.py +++ b/RepSys/commands/editlog.py @@ -30,7 +30,8 @@ def parse_options(): def editlog(pkgdirurl, revision): svn = SVN() - svn.propedit("svn:log", pkgdirurl, revision=revision, revprop=True) + svn.propedit("svn:log", pkgdirurl, revision=SVN.makerev(revision), + revprop=True) def main(): do_command(parse_options, editlog) diff --git a/RepSys/commands/markrelease.py b/RepSys/commands/markrelease.py index 9e52a31..440775b 100644 --- a/RepSys/commands/markrelease.py +++ b/RepSys/commands/markrelease.py @@ -9,7 +9,7 @@ # from RepSys import Error from RepSys.command import * -from RepSys.rpm import SRPM +from RepSys.simplerpm import SRPM from RepSys.rpmutil import mark_release from RepSys.util import get_auth import getopt diff --git a/RepSys/commands/submit.py b/RepSys/commands/submit.py index 6e9eb0f..5c95526 100644 --- a/RepSys/commands/submit.py +++ b/RepSys/commands/submit.py @@ -18,12 +18,18 @@ import xmlrpclib HELP = """\ Usage: repsys submit [OPTIONS] [URL [REVISION]] +Submits the package from URL to the submit host. + Options: -t TARGET Submit given package URL to given target -l Just list available targets -r REV Provides a revision number (when not providing as an argument) + -s The host in which the package URL will be submitted + (defaults to the host in the URL) -h Show this message + --define Defines one variable to be used by the submit scripts + in the submit host Examples: repsys submit @@ -39,6 +45,9 @@ def parse_options(): parser.add_option("-t", dest="target", default="Cooker") parser.add_option("-l", dest="list", action="store_true") parser.add_option("-r", dest="revision", type="string", nargs=1) + parser.add_option("-s", dest="submithost", type="string", nargs=1, + default=None) + parser.add_option("--define", action="append") opts, args = parser.parse_args() if not args: name, rev = get_submit_info(".") @@ -63,61 +72,33 @@ def parse_options(): raise Error, "provide -l or a revision number" return opts -def submit(pkgdirurl, revision, target, list=0): +def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None): #if not NINZ: # raise Error, "you must have NINZ installed to use this command" - type, rest = urllib.splittype(pkgdirurl) - host, path = urllib.splithost(rest) - user, host = urllib.splituser(host) - host, port = urllib.splitport(host) - if type != "https" and type != "svn+ssh": - raise Error, "you must use https:// or svn+ssh:// urls" - if user: - user, passwd = urllib.splitpasswd(user) - if passwd: - raise Error, "do not use a password in your command line" - if type == "https": - user, passwd = get_auth(username=user) - #soap = NINZ.client.Binding(host=host, - # url="https://%s/scripts/cnc/soap" % host, - # ssl=1, - # auth=(NINZ.client.AUTH.httpbasic, - # user, passwd)) - if port: - port = ":"+port - else: - port = "" - iface = xmlrpclib.ServerProxy("https://%s:%s@%s%s/scripts/cnc/xmlrpc" - % (user, passwd, host, port)) - try: - if list: - targets = iface.submit_targets() - if not targets: - raise Error, "no targets available" - sys.stdout.writelines(['"%s"\n' % x for x in targets]) - else: - iface.submit_package(pkgdirurl, revision, target) - print "Package submitted!" - #except NINZ.client.SoapError, e: - except xmlrpclib.ProtocolError, e: - raise Error, "remote error: "+str(e.errmsg) - except xmlrpclib.Fault, e: - raise Error, "remote error: "+str(e.faultString) - except xmlrpclib.Error, e: - raise Error, "remote error: "+str(e) + if submithost is None: + submithost = config.get("submit", "host") + if submithost is None: + # extract the submit host from the svn host + type, rest = urllib.splittype(pkgdirurl) + host, path = urllib.splithost(rest) + user, host = urllib.splituser(host) + submithost, port = urllib.splitport(host) + del type, user, port, path, rest + # runs a create-srpm in the server through ssh, which will make a + # copy of the rpm in the export directory + if list: + raise Error, "unable to list targets from svn+ssh:// URLs" + createsrpm = get_helper("create-srpm") + command = "ssh %s %s '%s' -r %s -t %s" % ( + submithost, createsrpm, pkgdirurl, revision, target) + if define: + command += " " + " ".join([ "--define " + x for x in define ]) + status, output = execcmd(command) + if status == 0: + print "Package submitted!" else: - # runs a create-srpm in the server through ssh, which will make a - # copy of the rpm in the export directory - if list: - raise Error, "unable to list targets from svn+ssh:// URLs" - createsrpm = get_helper("create-srpm") - command = "ssh %s %s '%s' -r %s -t %s" % ( - host, createsrpm, pkgdirurl, revision, target) - status, output = execcmd(command) - if status == 0: - print "Package submitted!" - else: - sys.exit(status) + sys.stderr.write(output) + sys.exit(status) def main(): diff --git a/RepSys/commands/sync.py b/RepSys/commands/sync.py new file mode 100644 index 0000000..42ede8d --- /dev/null +++ b/RepSys/commands/sync.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +from RepSys.command import * +from RepSys.rpmutil import sync + +HELP = """\ +Usage: repsys sync + +Will add or removed from the working copy new files added or removed +from the spec file. + +"No changes are commited." + +Options: + --dry-run Print results without changing the working copy + -h Show this message + +Examples: + repsys sync +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("--dry-run", dest="dryrun", default=False, + action="store_true") + opts, args = parser.parse_args() + if len(args): + opts.target = args[0] + return opts + +def main(): + do_command(parse_options, sync) diff --git a/RepSys/log.py b/RepSys/log.py index 5b33c6e..8035997 100644 --- a/RepSys/log.py +++ b/RepSys/log.py @@ -1,9 +1,12 @@ #!/usr/bin/python -from RepSys import Error, config +from RepSys import Error, config, RepSysTree from RepSys.svn import SVN from RepSys.util import execcmd -from Cheetah.Template import Template +try: + from Cheetah.Template import Template +except ImportError: + raise Error, "repsys requires the package python-cheetah" import sys import os @@ -15,8 +18,6 @@ import tempfile import shutil -locale.setlocale(locale.LC_ALL, "C") - default_template = """ #for $rel in $releases_by_author * $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release @@ -42,7 +43,7 @@ default_template = """ #end for """ -def getrelease(pkgdirurl, rev=None): +def getrelease(pkgdirurl, rev=None, macros=[]): """Tries to obtain the version-release of the package for a yet-not-markrelease revision of the package. @@ -50,28 +51,37 @@ def getrelease(pkgdirurl, rev=None): will be used. """ svn = SVN() + from RepSys.rpmutil import rpm_macros_defs tmpdir = tempfile.mktemp() try: - pkgname = os.path.basename(pkgdirurl) + pkgname = RepSysTree.pkgname(pkgdirurl) pkgcurrenturl = os.path.join(pkgdirurl, "current") specurl = os.path.join(pkgcurrenturl, "SPECS") if svn.exists(specurl): - svn.export(specurl, tmpdir, revision=SVN.revision(rev)) + svn.export(specurl, tmpdir, revision=SVN.makerev(rev)) found = glob.glob(os.path.join(tmpdir, "*.spec")) if found: specpath = found[0] - command = (("rpm -q --qf '%%{VERSION}-%%{RELEASE}\n' " - "--specfile %s") % specpath) + options = rpm_macros_defs(macros) + command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " + "--specfile %s %s 2>/dev/null") % + (specpath, options)) status, output = execcmd(command) if status != 0: raise Error, "Error in command %s: %s" % (command, output) releases = output.split() try: - version, release = releases[0].split("-", 1) + epoch, vr = releases[0].split(":", 1) + version, release = vr.split("-", 1) except ValueError: raise Error, "Invalid command output: %s: %s" % \ (command, output) - return version, release + #XXX check if this is the right way: + if epoch == "(none)": + ev = version + else: + ev = epoch + ":" + version + return ev, release finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) @@ -88,16 +98,34 @@ class ChangelogRevision: def __init__(self, **kwargs): self.__dict__.update(kwargs) + def __repr__(self): + lines = repr(self.lines)[:30] + "...]" + line = "" % \ + (self.revision, self.author, self.date, lines) + return line class ChangelogRelease(ChangelogRevision): version = None release = None - revisions = None + revisions = [] + release_revisions = [] + authors = [] + visible = False def __init__(self, **kwargs): ChangelogRevision.__init__(self, **kwargs) self.revisions = [] + def __repr__(self): + line = "" % \ + (self.version, self.release, self.revisions) + return line + +unescaped_macro_pat = re.compile(r"([^%])%([^%])") + +def escape_macros(text): + escaped = unescaped_macro_pat.sub("\\1%%\\2", text) + return escaped def format_lines(lines): first = 1 @@ -105,7 +133,7 @@ def format_lines(lines): perexpr = re.compile(r"([^%])%([^%])") for line in lines: if line: - line = perexpr.sub("\\1%%\\2", line) + line = escape_macros(line) if first: first = 0 line = line.lstrip() @@ -130,8 +158,10 @@ class ChangelogByAuthor: def group_releases_by_author(releases): allauthors = [] + grouped = [] for release in releases: authors = {} + latest = None for revision in release.revisions: authors.setdefault(revision.author, []).append(revision) @@ -144,60 +174,114 @@ def group_releases_by_author(releases): revdeco = [(r.revision, r) for r in revs] revdeco.sort(reverse=1) author.revisions = [t[1] for t in revdeco] - decorated.append((max(revdeco)[0], author)) + revlatest = author.revisions[0] + # keep the latest revision even for silented authors (below) + if latest is None or revlatest.revision > latest.revision: + latest = revlatest + count = sum(len(rev.lines) for rev in author.revisions) + if count == 0: + # skipping author with only silented lines + continue + decorated.append((revdeco[0][0], author)) + + if not decorated: + # skipping release with only authors with silented lines + continue decorated.sort(reverse=1) release.authors = [t[1] for t in decorated] - # the difference between a released and a not released - # ChangelogRelease is the way the release numbers is obtained. So, - # when this is a released, we already have it, but if we don't, we - # should get de version/release string using getrelease and then - # get the first + # the difference between a released and a not released _Release is + # the way the release numbers is obtained. So, when this is a + # released, we already have it, but if we don't, we should get de + # version/release string using getrelease and then get the first first, release.authors = release.authors[0], release.authors[1:] release.author_name = first.name release.author_email = first.email - release.date = first.revisions[0].date - release.raw_date = first.revisions[0].raw_date release.release_revisions = first.revisions - release.revision = first.revisions[0].revision - return releases - + #release.date = first.revisions[0].date + release.date = latest.date + release.raw_date = latest.raw_date + #release.revision = first.revisions[0].revision + release.revision = latest.revision + + grouped.append(release) + + return grouped + + +def group_revisions_by_author(currentlog): + revisions = [] + last_author = None + for entry in currentlog: + revision = ChangelogRevision() + revision.lines = format_lines(entry.lines) + revision.raw_date = entry.date + revision.date = parse_raw_date(entry.date) + revision.revision = entry.revision + if entry.author == last_author: + revisions[-1].revisions.append(revision) + else: + author = ChangelogByAuthor() + author.name, author.email = get_author_name(entry.author) + author.revisions = [revision] + revisions.append(author) + last_author = entry.author + return revisions + emailpat = re.compile("(?P.*?)\s*<(?P.*?)>") +def get_author_name(author): + found = emailpat.match(config.get("users", author, author)) + name = ((found and found.group("name")) or author) + email = ((found and found.group("email")) or author) + return name, email + +def parse_raw_date(rawdate): + return time.strftime("%a %b %d %Y", rawdate) + +def filter_log_lines(lines): + # lines in commit messages containing SILENT at any position will be + # skipped; commits with their log messages beggining with SILENT in the + # first positionj of the first line will have all lines ignored. + ignstr = config.get("log", "ignore-string", "SILENT") + if len(lines) and lines[0].startswith(ignstr): + return [] + filtered = [line for line in lines if ignstr not in line] + return filtered + def make_release(author=None, revision=None, date=None, lines=None, entries=[], released=True, version=None, release=None): rel = ChangelogRelease() rel.author = author - found = emailpat.match(config.get("users", author, author or "")) - rel.author_name = (found and found.group("name")) or author - rel.author_email = (found and found.group("email")) or author + if author: + rel.author_name, rel.author_email = get_author_name(author) rel.revision = revision rel.version = version rel.release = release - rel.date = (date and time.strftime("%a %b %d %Y", date)) or None + rel.date = (date and parse_raw_date(date)) or None rel.lines = lines rel.released = released + rel.visible = False for entry in entries: + lines = filter_log_lines(entry.lines) + if lines: + rel.visible = True revision = ChangelogRevision() revision.revision = entry.revision - revision.lines = format_lines(entry.lines) - revision.date = time.strftime("%a %b %d %Y", entry.date) + revision.lines = format_lines(lines) + revision.date = parse_raw_date(entry.date) revision.raw_date = entry.date revision.author = entry.author - found = emailpat.match(config.get("users", entry.author, entry.author)) - revision.author_name = ((found and found.group("name")) or - entry.author) - revision.author_email = ((found and found.group("email")) or - entry.author) + (revision.author_name, revision.author_email) = \ + get_author_name(entry.author) rel.revisions.append(revision) return rel -def dump_file(releases, template=None): - +def dump_file(releases, currentlog=None, template=None): templpath = template or config.get("template", "path", None) params = {} if templpath is None or not os.path.exists(templpath): @@ -207,10 +291,12 @@ def dump_file(releases, template=None): else: params["file"] = templpath releases_author = group_releases_by_author(releases) + revisions_author = group_revisions_by_author(currentlog) params["searchList"] = [{"releases_by_author" : releases_author, - "releases" : releases}] + "releases" : releases, + "revisions_by_author": revisions_author}] t = Template(**params) - return repr(t) + return t.respond() class InvalidEntryError(Exception): @@ -249,8 +335,30 @@ def get_revision_offset(): "file(s).") return revoffset or 0 +oldmsgpat = re.compile( + r"Copying release (?P[^\s]+) to (?P[^\s]+) directory\.") -def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, template=None): +def parse_markrelease_log(relentry): + if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \ + or parse_repsys_entry(relentry))): + raise InvalidEntryError + from_rev = None + path = None + for changed in relentry.changed: + if changed["action"] == "A" and changed["from_rev"]: + from_rev = changed["from_rev"] + path = changed["path"] + break + else: + raise InvalidEntryError + # get the version and release from the names in the path, do not relay + # on log messages + version, release = path.rsplit(os.path.sep, 3)[-2:] + return version, release, from_rev + + +def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, + template=None, macros=[]): size = size or 0 concat = config.get("log", "concat", "").split() revoffset = get_revision_offset() @@ -261,68 +369,74 @@ def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, template=None): strict_node_history=False, noerror=1)) or [] currentlog = list(svn.log(pkgcurrenturl, strict_node_history=False, - revision_start=SVN.revision(rev), - revision_end=SVN.revision(revoffset), limit=size)) - lastauthor = None - previous_revision = 0 - currelease = None + revision_start=SVN.makerev(rev), + revision_end=SVN.makerev(revoffset), limit=size)) + + # sort releases by copyfrom-revision, so that markreleases for same + # revisions won't look empty + releasesdata = [] + if releaseslog: + for relentry in releaseslog[::-1]: + try: + (version, release, relrevision) = \ + parse_markrelease_log(relentry) + except InvalidEntryError: + continue + releasesdata.append((relrevision, -relentry.revision, relentry, + version, release)) + releasesdata.sort() + + # collect valid releases using the versions provided by the changes and + # the packages + prevrevision = 0 releases = [] - - # for the emergency bug fixer: the [].sort() is done using the - # decorate-sort-undecorate pattern - releases_data = [] - for relentry in releaseslog[::-1]: - try: - revinfo = parse_repsys_entry(relentry) - except InvalidEntryError: + for (relrevision, dummy, relentry, version, release) in releasesdata: + if prevrevision == relrevision: + # ignore older markrelease of the same revision, since they + # will have no history continue - try: - release_number = int(revinfo["revision"]) - except (KeyError, ValueError): - raise Error, "Error parsing data from log entry from r%s" % \ - relentry.revision - releases_data.append((release_number, relentry, revinfo)) - releases_data.sort() - - for release_number, relentry, revinfo in releases_data: - # get entries newer than 'previous' and older than 'relentry' entries = [entry for entry in currentlog - if release_number >= entry.revision and - (previous_revision < entry.revision)] + if relrevision >= entry.revision and + (prevrevision < entry.revision)] if not entries: #XXX probably a forced release, without commits in current/, - # check if this is the right behavior and if some release is - # not being lost. + # check if this is the right behavior + sys.stderr.write("warning: skipping (possible) release " + "%s-%s@%s, no commits since previous markrelease (r%r)\n" % + (version, release, relrevision, prevrevision)) continue release = make_release(author=relentry.author, revision=relentry.revision, date=relentry.date, lines=relentry.lines, entries=entries, - version=revinfo["version"], release=revinfo["release"]) + version=version, release=release) releases.append(release) - previous_revision = release_number - + prevrevision = relrevision + # look for commits that have been not submited (released) yet # this is done by getting all log entries newer (revision larger) - # than releaseslog[0] - latest_revision = releaseslog[0].revision + # than releaseslog[0] (in the case it exists) + if releaseslog: + latest_revision = releaseslog[0].revision + else: + latest_revision = 0 notsubmitted = [entry for entry in currentlog if entry.revision > latest_revision] if notsubmitted: # if they are not submitted yet, what we have to do is to add # a release/version number from getrelease() - version, release = getrelease(pkgdirurl) + version, release = getrelease(pkgdirurl, macros=macros) toprelease = make_release(entries=notsubmitted, released=False, version=version, release=release) releases.append(toprelease) - data = dump_file(releases[::-1], template=template) + data = dump_file(releases[::-1], currentlog=currentlog, template=template) return data def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, - submit=False, template=None): + submit=False, template=None, macros=[]): newlines = [] found = 0 @@ -339,7 +453,7 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, # Create new changelog newlines.append("\n\n%changelog\n") newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, - template=template)) + template=template, macros=macros)) # Merge old changelog, if available oldurl = config.get("log", "oldurl") @@ -347,15 +461,19 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, svn = SVN() tmpdir = tempfile.mktemp() try: - pkgname = os.path.basename(pkgdirurl) + pkgname = RepSysTree.pkgname(pkgdirurl) pkgoldurl = os.path.join(oldurl, pkgname) if svn.exists(pkgoldurl): - svn.export(pkgoldurl, tmpdir, rev=rev) + # we're using HEAD here because fixes in misc/ (oldurl) may + # be newer than packages' last changed revision. + svn.export(pkgoldurl, tmpdir) logfile = os.path.join(tmpdir, "log") if os.path.isfile(logfile): file = open(logfile) newlines.append("\n") - newlines.append(file.read()) + log = file.read() + log = escape_macros(log) + newlines.append(log) file.close() finally: if os.path.isdir(tmpdir): diff --git a/RepSys/mirror.py b/RepSys/mirror.py new file mode 100644 index 0000000..a24f594 --- /dev/null +++ b/RepSys/mirror.py @@ -0,0 +1,42 @@ +import os +import urlparse + +from RepSys import config +from RepSys.svn import SVN + +def relocate_path(oldparent, newparent, url): + subpath = url[len(oldparent)-1:] + newurl = newparent + "/" + subpath # subpath usually gets / at begining + return newurl + +def enabled(): + mirror = config.get("global", "mirror") + default_parent = config.get("global", "default_parent") + return (mirror is not None and + default_parent is not None) + +def mirror_relocate(oldparent, newparent, url, wcpath): + svn = SVN(noauth=True) + newurl = relocate_path(oldparent, newparent, url) + svn.switch(newurl, url, path=wcpath, relocate="True") + return newurl + +def switchto_parent(svn, url, path): + """Relocates the working copy to default_parent""" + mirror = config.get("global", "mirror") + default_parent = config.get("global", "default_parent") + newurl = mirror_relocate(mirror, default_parent, url, path) + return newurl + +def switchto_mirror(svn, url, path): + mirror = config.get("global", "mirror") + default_parent = config.get("global", "default_parent") + newurl = mirror_relocate(default_parent, mirror, url, path) + return newurl + +def checkout_url(url): + mirror = config.get("global", "mirror") + default_parent = config.get("global", "default_parent") + if mirror is not None and default_parent is not None: + return relocate_path(default_parent, mirror, url) + return url diff --git a/RepSys/plugins/__init__.py b/RepSys/plugins/__init__.py new file mode 100644 index 0000000..e4f4e08 --- /dev/null +++ b/RepSys/plugins/__init__.py @@ -0,0 +1,27 @@ +import os + +loaded = {} + +def load(): + # based on smart's plugin system + pluginsdir = os.path.dirname(__file__) + for entry in os.listdir(pluginsdir): + if entry != "__init__.py" and entry.endswith(".py"): + name = entry[:-3] + loaded[name] = __import__("RepSys.plugins."+name, {}, {}, + [name]) + elif os.path.isdir(entry): + initfile = os.path.join(entry, "__init__.py") + if os.path.isfile(initfile): + loaded[entry] = __import__("RepSys.plugins."+entry, {}, {}, + [entry]) + +def list(): + return loaded.keys() + +def help(name): + from RepSys import Error + try: + return loaded[name].__doc__ + except KeyError: + raise Error, "plugin %s not found" % name diff --git a/RepSys/plugins/ldapusers.py b/RepSys/plugins/ldapusers.py new file mode 100644 index 0000000..75d362c --- /dev/null +++ b/RepSys/plugins/ldapusers.py @@ -0,0 +1,164 @@ +""" +A Repsys plugin for obtaining users from a LDAP server. + +In order to enable the plugin, the user must define the following +options in the [global] section of repsys.conf: + + ldap-server [required] + the host name of the LDAP server + ldap-port [optional] [default: 389] + the port of the LDAP server + ldap-base [required] + the base DN where the search will be performed + ldap-binddn [optional] [default: empty] + the DN used to bind + ldap-bindpw [optional] [default: empty] + the password used to bind + ldap-filterformat [optional] + [default: (&(objectClass=inetOrgPerson)(uid=$username))] + RFC-2254 filter string used in the search of the user entry. + Note that this is a python template string and will have the + user name as parameter. For example: + + ldap-filterformat = (&(objectClass=inetOrgPerson)(uid=$username)) + + Will result in the search filter: + + (&(objectClass=inetOrgPerson)(uid=john)) + + ldap-resultformat [optional] [default: $cn <$mail>] + This is a python template string. This string will be + formatted using one dict object containing the fields + returned in the LDAP search, for example: + + >>> format = Template("$cn <$mail>") + >>> d = search(basedn, filter) + >>> d + {"cn": "John Doe", "mail": "john@mandriva.org", + "uidNumber": "1290", "loginShell": "/bin/bash", + ... many other attributes ... } + >>> value = format.substitute(d) + >>> print value + John Doe + + Note that only the first value of the attributes will be + used. + +When the searched option is not found, it will try in repsys.conf. All +the values found. (including from repsys.conf) will be cached between +each configuration access. + +This plugin requires the package python-ldap. + +For more information, look http://qa.mandriva.com/show_bug.cgi?id=30549 +""" +from RepSys import Error, config + +import string + +users_cache = {} + +class LDAPError(Error): + def __init__(self, ldaperr): + self.ldaperr = ldaperr + name = ldaperr.__class__.__name__ + desc = ldaperr.message["desc"] + self.message = "LDAP error %s: %s" % (name, desc) + self.args = self.message, + +def strip_entry(entry): + "Leave only the first value in all keys in the entry" + new = dict((key, value[0]) for key, value in entry.iteritems()) + return new + +def interpolate(optname, format, data): + tmpl = string.Template(format) + try: + return tmpl.substitute(data) + except KeyError, e: + raise Error, "the key %s was not found in LDAP search, " \ + "check your %s configuration" % (e, optname) + except (TypeError, ValueError), e: + raise Error, "LDAP response formatting error: %s. Check " \ + "your %s configuration" % (e, optname) + +def used_attributes(format): + class DummyDict: + def __init__(self): + self.found = [] + def __getitem__(self, key): + self.found.append(key) + return key + dd = DummyDict() + t = string.Template(format) + t.safe_substitute(dd) + return dd.found + +def make_handler(): + server = config.get("global", "ldap-server") + try: + port = int(config.get("global", "ldap-port", 389)) + except ValueError: + raise Error, "the option ldap-port requires an integer, please "\ + "check your configuration files" + basedn = config.get("global", "ldap-base") + binddn = config.get("global", "ldap-binddn") + bindpw = config.get("global", "ldap-bindpw", "") + filterformat = config.get("global", "ldap-filterformat", + "(&(objectClass=inetOrgPerson)(uid=$username))", raw=1) + format = config.get("global", "ldap-resultformat", "$cn <$mail>", raw=1) + + if server is None: + def dummy_wrapper(section, option=None, default=None, walk=False): + return config.get(section, option, default, wrap=False) + return dummy_wrapper + + try: + import ldap + except ImportError: + raise Error, "LDAP support needs the python-ldap package "\ + "to be installed" + else: + from ldap.filter import escape_filter_chars + + def users_wrapper(section, option=None, default=None, walk=False): + global users_cache + if walk: + raise Error, "ldapusers plugin does not support user listing" + assert option is not None, \ + "When not section walking, option is required" + + value = users_cache.get(option) + if value is not None: + return value + + try: + l = ldap.open(server, port) + if binddn: + l.bind(binddn, bindpw) + except ldap.LDAPError, e: + raise LDAPError(e) + try: + data = {"username": escape_filter_chars(option)} + filter = interpolate("ldap-filterformat", filterformat, data) + attrs = used_attributes(format) + try: + found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter, + attrlist=attrs) + except ldap.LDAPError, e: + raise LDAPError(e) + if found: + dn, entry = found[0] + entry = strip_entry(entry) + value = interpolate("ldap-resultformat", format, entry) + else: + # issue a warning? + value = config.get(section, option, default, wrap=False) + users_cache[option] = value + return value + finally: + l.unbind_s() + + return users_wrapper + +config.wrap("users", handler=make_handler()) diff --git a/RepSys/plugins/sample.py.txt b/RepSys/plugins/sample.py.txt new file mode 100644 index 0000000..9877f3c --- /dev/null +++ b/RepSys/plugins/sample.py.txt @@ -0,0 +1,14 @@ +# Sample repsys plugin. In order to test it, rename to sample.py +# vim:ft=python +from RepSys import config + +def users_wrapper(section, option=None, default=None, walk=False): + d = {"foolano": "Foolano De Tal ", + "ceeclano": "Ceeclano Algumacoisa ", + "beltrano": "Beltrano Bla "} + if walk: + return d.items() + + return d.get(option, default) + +config.wrap("users", handler=users_wrapper) diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py index 8b8c8b7..39e78fc 100644 --- a/RepSys/rpmutil.py +++ b/RepSys/rpmutil.py @@ -1,10 +1,12 @@ #!/usr/bin/python -from RepSys import Error, config +from RepSys import Error, config, RepSysTree +from RepSys import mirror from RepSys.svn import SVN -from RepSys.rpm import SRPM +from RepSys.simplerpm import SRPM from RepSys.log import specfile_svn2rpm from RepSys.util import execcmd import pysvn +import rpm import tempfile import shutil import glob @@ -26,6 +28,11 @@ def get_spec(pkgdirurl, targetdir=".", submit=False): if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) +def rpm_macros_defs(macros): + defs = ("--define \"%s %s\"" % macro for macro in macros) + args = " ".join(defs) + return args + def get_srpm(pkgdirurl, mode = "current", targetdirs = None, @@ -38,6 +45,7 @@ def get_srpm(pkgdirurl, scripts = [], submit = False, template = None, + macros = [], verbose = 0): svn = SVN() tmpdir = tempfile.mktemp() @@ -58,7 +66,7 @@ def get_srpm(pkgdirurl, geturl = os.path.join(pkgdirurl, "current") else: raise Error, "unsupported get_srpm mode: %s" % mode - svn.checkout(geturl, tmpdir, revision=SVN.revision(revision)) + svn.export(geturl, tmpdir, revision=SVN.makerev(revision)) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") @@ -69,9 +77,12 @@ def get_srpm(pkgdirurl, if svnlog: submit = not not revision specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, - template=template) - revisionreal = svn.info(tmpdir).revision.number + template=template, macros=macros) + #FIXME revisioreal not needed if revision is None + #FIXME use geturl instead of pkgdirurl + revisionreal = svn.revision(pkgdirurl) for script in scripts: + #FIXME revision can be "None" status, output = execcmd(script, tmpdir, spec, str(revision), noerror=1) if status != 0: @@ -79,11 +90,13 @@ def get_srpm(pkgdirurl, if packager: packager = " --define 'packager %s'" % packager - execcmd("rpm -bs --nodeps %s %s %s %s %s %s %s %s %s" % + defs = rpm_macros_defs(macros) + execcmd("rpm -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % (topdir, builddir, rpmdir, sourcedir, specdir, - srcrpmdir, patchdir, packager, spec)) + srcrpmdir, patchdir, packager, spec, defs)) if revision and revisionreal: + #FIXME duplicate glob line srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0] srpminfo = SRPM(srpm) release = srpminfo.release @@ -94,7 +107,8 @@ def get_srpm(pkgdirurl, targetdirs = (".",) targetsrpms = [] for targetdir in targetdirs: - targetsrpm = os.path.join(os.path.realpath(targetdir), os.path.basename(srpm)) + targetsrpm = os.path.join(os.path.realpath(targetdir), + os.path.basename(srpm)) targetsrpms.append(targetsrpm) if verbose: sys.stderr.write("Wrote: %s\n" % targetsrpm) @@ -226,7 +240,7 @@ def create_package(pkgdirurl, log="", verbose=0): svn = SVN() tmpdir = tempfile.mktemp() try: - basename = os.path.basename(pkgdirurl) + basename = RepSysTree.pkgname(pkgdirurl) if verbose: print "Creating package directory...", sys.stdout.flush() @@ -269,7 +283,7 @@ def mark_release(pkgdirurl, version, release, revision): versionurl = "/".join([releasesurl, version]) releaseurl = "/".join([versionurl, release]) if svn.exists(releaseurl, noerror=1): - raise cncrep.Error, "release already exists" + raise Error, "release already exists" svn.mkdir(releasesurl, noerror=1, log="Created releases directory.") svn.mkdir(versionurl, noerror=1, @@ -282,13 +296,13 @@ def mark_release(pkgdirurl, version, release, revision): log="Copying release %s-%s to pristine/ directory." % (version, release)) markreleaselog = create_markrelease_log(version, release, revision) - svn.copy(currenturl, releaseurl, rev=revision, + svn.copy(currenturl, releaseurl, revision=revision, log=markreleaselog) -def check_changed(url, all=0, show=0, verbose=0): +def check_changed(pkgdirurl, all=0, show=0, verbose=0): svn = SVN() if all: - baseurl = url + baseurl = pkgdirurl packages = [] if verbose: print "Getting list of packages...", @@ -299,7 +313,7 @@ def check_changed(url, all=0, show=0, verbose=0): if not packages: raise Error, "couldn't get list of packages" else: - baseurl, basename = os.path.split(url) + baseurl, basename = os.path.split(pkgdirurl) packages = [basename] clean = [] changed = [] @@ -332,22 +346,103 @@ def check_changed(url, all=0, show=0, verbose=0): if verbose: print "clean" clean.append(package) - if verbose and all: - print "Total clean packages: %s" % len(clean) - print "Total CHANGED packages: %d" % len(changed) - print "Total NO CURRENT packages: %s" % len(nocurrent) - print "Total NO PRISTINE packages: %s" % len(nopristine) + if verbose: + if not packages: + print "No packages found!" + elif all: + print "Total clean packages: %s" % len(clean) + print "Total CHANGED packages: %d" % len(changed) + print "Total NO CURRENT packages: %s" % len(nocurrent) + print "Total NO PRISTINE packages: %s" % len(nopristine) return {"clean": clean, "changed": changed, "nocurrent": nocurrent, "nopristine": nopristine} -def checkout(url, path=None, revision=None): +def checkout(pkgdirurl, path=None, revision=None): svn = SVN() - current = os.path.join(url, "current") + current = os.path.join(pkgdirurl, "current") if path is None: - _, path = os.path.split(url) - svn.checkout(current, path, revision=SVN.revision(revision), show=1) + _, path = os.path.split(pkgdirurl) + if mirror.enabled(): + current = mirror.checkout_url(current) + print "checking out from mirror", current + svn.checkout(current, path, revision=SVN.makerev(revision), show=1) + +def sync(dryrun=False): + svn = SVN() + cwd = os.getcwd() + dirname = os.path.basename(cwd) + if dirname == "SPECS" or dirname == "SOURCES": + topdir = os.pardir + else: + topdir = "" + # run svn info because svn st does not complain when topdir is not an + # working copy + svn.info(topdir or ".") + specsdir = os.path.join(topdir, "SPECS/") + sourcesdir = os.path.join(topdir, "SOURCES/") + for path in (specsdir, sourcesdir): + if not os.path.isdir(path): + raise Error, "%s directory not found" % path + specs = glob.glob(os.path.join(specsdir, "*.spec")) + if not specs: + raise Error, "no .spec files found in %s" % specsdir + specpath = specs[0] # FIXME better way? + try: + spec = rpm.TransactionSet().parseSpec(specpath) + except rpm.error, e: + raise Error, "could not load spec file: %s" % e + sources = [os.path.basename(name) + for name, no, flags in spec.sources()] + sourcesst = dict((os.path.basename(st.path), st) + for st in svn.status(sourcesdir, get_all=False, ignore=True)) + toadd = [] + for source in sources: + sourcepath = os.path.join(sourcesdir, source) + if sourcesst.get(source): + if os.path.isfile(sourcepath): + toadd.append(sourcepath) + else: + sys.stderr.write("warning: %s not found\n" % sourcepath) + # rm entries not found in sources and still in svn + found = os.listdir(sourcesdir) + toremove = [] + for entry in found: + if entry == ".svn": + continue + status = sourcesst.get(entry) + if status is None and entry not in sources: + path = os.path.join(sourcesdir, entry) + toremove.append(path) + for path in toremove: + print "D\t%s" % path + if not dryrun: + svn.remove(path) + for path in toadd: + print "A\t%s" % path + if not dryrun: + svn.add(path) + +def commit(target=".", message=None): + svn = SVN() + info = svn.info(target) + url = info.url + if url is None: + raise Error, "working copy URL not provided by svn info" + if mirror.enabled(): + newurl = mirror.switchto_parent(svn, url, target) + print "relocated to", newurl + try: + # we can't use the svn object here because pexpect hides VISUAL + mopt = "" + if message is not None: + mopt = "-m \"%s\"" % message + os.system("svn ci %s %s" % (mopt, target)) + finally: + if mirror.enabled(): + mirror.switchto_mirror(svn, newurl, target) + print "relocated back to", url def get_submit_info(path): path = os.path.abspath(path) diff --git a/RepSys/svn.py b/RepSys/svn.py index f055d8a..2fe9ad1 100644 --- a/RepSys/svn.py +++ b/RepSys/svn.py @@ -19,6 +19,7 @@ class SVNLogEntry: self.revision = revision self.author = author self.date = date + self.changed = [] self.lines = [] def __cmp__(self, other): @@ -33,25 +34,42 @@ class SVN: self._client = pysvn.Client() self._client_lock = threading.Lock() self._client.callback_get_log_message = self._log_handler + self._client.callback_get_login = self._unsupported_auth + self._client.callback_ssl_client_cert_password_prompt = \ + self._unsupported_auth + self._client.callback_ssl_client_cert_prompt = \ + self._unsupported_auth + self._client.callback_ssl_server_prompt = \ + self._unsupported_auth def _log_handler(self): if self._current_message is None: + #TODO make it use EDITOR raise ValueError, "No log message defined" return True, self._current_message + def _unsupported_auth(self, *args, **kwargs): + raise SVNError, "svn is trying to get login information, " \ + "seems that you're not using ssh-agent" + def _get_log_message(self, received_kwargs): message = received_kwargs.pop("log", None) messagefile = received_kwargs.pop("logfile", None) if messagefile and not message: message = open(messagefile).read() return message - - def _make_wrapper(self, meth): + + def _set_notify_callback(self, callback): + self._client.callback_notify = callback + + def _make_wrapper(self, meth, notify=None): def wrapper(*args, **kwargs): self._client_lock.acquire() try: self._current_message = self._get_log_message(kwargs) ignore_errors = kwargs.pop("noerror", None) + if notify: + self._client.callback_notify = notify try: return meth(*args, **kwargs) except pysvn.ClientError, (msg,): @@ -59,41 +77,63 @@ class SVN: raise SVNError, msg return None finally: - self._current_message = None self._client_lock.release() + self._current_message = None return wrapper - def __getattr__(self, attrname): + def _client_wrap(self, attrname): meth = getattr(self._client, attrname) wrapper = self._make_wrapper(meth) return wrapper - def revision(number=None, head=None): + def __getattr__(self, attrname): + return self._client_wrap(attrname) + + def makerev(number=None, head=None): if number is not None: args = (pysvn.opt_revision_kind.number, number) else: args = (pysvn.opt_revision_kind.head,) return pysvn.Revision(*args) - revision = staticmethod(revision) + makerev = staticmethod(makerev) + + def revision(self, url): + infos = self._client.info2(url, recurse=False) + revnum= infos[0][1].rev.number + return revnum # this override method fixed the problem in pysvn's mkdir which # requires a log_message parameter def mkdir(self, path, log=None, **kwargs): - meth = self.__getattr__("mkdir") + meth = self._client_wrap("mkdir") # we can't raise an error because pysvn's mkdir will use # log_message only if path is remote, but it *always* requires this # parameter. Also, 'log' is never used. log = log or "There's a silent bug in your code" return meth(path, log, log=None, **kwargs) + + def checkout(self, url, targetpath, show=False, **kwargs): + if show: + def callback(event): + types = pysvn.wc_notify_action + action = event["action"] + if action == types.update_add: + print "A %s" % event["path"] + elif action == types.update_completed: + print "Checked out revision %d" % \ + event["revision"].number + self._set_notify_callback(callback) + meth = self._client_wrap("checkout") + meth(url, targetpath, **kwargs) def checkin(self, path, log, **kwargs): # XXX use EDITOR when log empty - meth = self.__getattr__("checkin") + meth = self._client_wrap("checkin") return meth(path, log, log=None, **kwargs) - + def log(self, *args, **kwargs): - meth = self.__getattr__("log") - entries = meth(*args, **kwargs) + meth = self._client_wrap("log") + entries = meth(discover_changed_paths=True, *args, **kwargs) if entries is None: return for entrydic in entries: @@ -101,11 +141,32 @@ class SVN: entrydic["author"], time.localtime(entrydic["date"])) entry.lines[:] = entrydic["message"].split("\n") + for cp in entrydic["changed_paths"]: + from_rev = cp["copyfrom_revision"] + if from_rev: + from_rev = from_rev.number + changed = { + "action": cp["action"], + "path": cp["path"], + "from_rev": from_rev, + "from_path": cp["copyfrom_path"], + } + entry.changed.append(changed) yield entry - + def exists(self, path): return self.ls(path, noerror=1) is not None + def diff(self, *args, **kwargs): + head = pysvn.Revision(pysvn.opt_revision_kind.head) + revision1 = kwargs.pop("revision1", head) + revision2 = kwargs.pop("revision2", head) + tmpdir = tempfile.gettempdir() + meth = self._client_wrap("diff") + diff_text = meth(tmpdir, revision1=revision1, revision2=revision2, + *args, **kwargs) + return diff_text + def _edit_message(self, message): # argh! editor = os.getenv("EDITOR", "vim") @@ -116,8 +177,10 @@ class SVN: f.write(message) f.close() lastchange = os.stat(fpath).st_mtime - while 1: - os.system("%s %s" % (editor, fpath)) + for i in xrange(10): + status = os.system("%s %s" % (editor, fpath)) + if status != 0: + raise SVNError, "the editor failed with %d" % status newchange = os.stat(fpath).st_mtime if newchange == lastchange: print "Log message unchanged or not specified" @@ -137,7 +200,7 @@ class SVN: return result def propedit(self, propname, pkgdirurl, revision, revprop=False): - revision = self.revision(revision) + revision = (revision) if revprop: propget = self.revpropget propset = self.revpropset diff --git a/TODO.LDAP b/TODO.LDAP new file mode 100644 index 0000000..54d6350 --- /dev/null +++ b/TODO.LDAP @@ -0,0 +1,4 @@ +- we should have a generic fqdn here to use round-robin DNS + enhancement for repsys: support multiple ldap servers here + (from repsys.conf in kenobi) +- support STARTTLS diff --git a/create-srpm b/create-srpm index 544b011..3dab068 100755 --- a/create-srpm +++ b/create-srpm @@ -1,6 +1,6 @@ #!/usr/bin/python -from RepSys import Error, config +from RepSys import Error, config, plugins from RepSys.rpmutil import get_srpm from RepSys.cgiutil import get_targets from RepSys.util import mapurl, execcmd, get_helper @@ -52,7 +52,8 @@ class CmdIface: packager=packager, revname=1, svnlog=1, - scripts=target.scripts) + scripts=target.scripts, + macros=target.macros) uploadsrpm = get_helper("upload-srpm") if uploadsrpm: @@ -63,8 +64,12 @@ class CmdIface: upload_command.append(x) upload_command.append(targetname) upload_command.append(targetsrpms[0]) - status, output = execcmd(" ".join(upload_command)) - os.unlink(targetsrpms[0]) + status, output = execcmd(" ".join(upload_command), + noerror=1) + if os.path.isfile(targetsrpms[0]): + os.unlink(targetsrpms[0]) + else: + sys.stderr.write("warning: upload ok; temp file '%s' removed unexpectedly\n" % (targetsrpms[0])) if status != 0: raise CmdError, "Failed to upload %s:\n%s" % (packageurl, output) return 1 @@ -91,9 +96,14 @@ def parse_options(): def main(): + plugins.load() iface = CmdIface() opts, args = parse_options() - iface.submit_package(args[0], opts.revision, opts.target, opts.urlmap, opts.define) + try: + iface.submit_package(args[0], opts.revision, opts.target, opts.urlmap, opts.define) + except Error, e: + sys.stderr.write("error: %s\n" % str(e)) + sys.exit(1) if __name__ == "__main__": diff --git a/default.chlog b/default.chlog index 4b767cf..aff3958 100644 --- a/default.chlog +++ b/default.chlog @@ -3,8 +3,11 @@ #for $rel in $releases_by_author * $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release + Revision: $rel.revision - #if not $rel.released -+ Status: not released +## #if not $rel.released +##+ Status: not released +## #end if + #if not $rel.visible ++ rebuild (emptylog) #end if #for $rev in $rel.release_revisions #for $line in $rev.lines @@ -13,6 +16,13 @@ $line #end for #for $author in $rel.authors + #if $author.revisions and not $author.revisions[0].lines + #continue + #end if + ##alternatively, one could use: + ###if $author.email == "root" + ## #continue + ###end if + $author.name <$author.email> #for $rev in $author.revisions #for $line in $rev.lines diff --git a/repsys b/repsys index e800a7f..804ad05 100755 --- a/repsys +++ b/repsys @@ -1,18 +1,21 @@ #!/usr/bin/python -from RepSys import Error +from RepSys import Error, plugins from RepSys.command import * import getopt import sys +import codecs +import locale VERSION="1.7.r%s" % ("$Rev$".split()[-2].strip()) HELP = """\ -$Id$ Usage: repsys COMMAND [COMMAND ARGUMENTS] Useful commands: co + sync + ci submit create getspec @@ -20,17 +23,36 @@ Useful commands: rpmlog changed authoremail + putsrpm Run "repsys COMMAND --help" for more information. +Run "repsys --help-plugins" for help on loaded plugins. + Written by Gustavo Niemeyer """ +def plugin_help(opt, val, parser, mode): + if parser is None: + prog = sys.argv[0] + print "Use %s --help-plugin " % prog + print "Available plugins:" + print + for name in plugins.list(): + print name + else: + print plugins.help(parser) + raise SystemExit + def parse_options(): parser = OptionParser(help=HELP, version="%prog "+VERSION) parser.disable_interspersed_args() parser.add_option("--debug", action="store_true") + parser.add_option("--help-plugins", action="callback", callback=plugin_help) + parser.add_option("--help-plugin", type="string", dest="__ignore", + action="callback", callback=plugin_help) opts, args = parser.parse_args() + del opts.__ignore if len(args) < 1: parser.print_help(sys.stderr) sys.exit(1) @@ -53,6 +75,14 @@ def dispatch_command(command, argv, debug=0): command_module.main() if __name__ == "__main__": + try: + plugins.load() + except Error, e: + sys.stderr.write("plugin initialization error: %s\n" % e) + sys.exit(1) + encoding = locale.getpreferredencoding() + sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors="replace") + sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors="replace") do_command(parse_options, dispatch_command) # vim:et:ts=4:sw=4 diff --git a/repsys.conf b/repsys.conf index 5679c87..bbe99af 100644 --- a/repsys.conf +++ b/repsys.conf @@ -2,6 +2,8 @@ verbose = no default_parent = svn+ssh://svn.mandriva.com/svn/packages/cooker url-map = svn\+ssh://svn\.mandriva\.com/(.*) file:///\1 +#mirror = http://svn.mandriva.com/svn/packages/cooker/ +#tempdir = /tmp [log] oldurl = svn+ssh://svn.mandriva.com/svn/packages/misc @@ -9,27 +11,42 @@ oldurl = svn+ssh://svn.mandriva.com/svn/packages/misc # will be constructed (default zero, i.e., oldest # commit) revision-offset = 0 +# commit lines containing this string won't be shown in the changelog: +ignore-string = SILENT [template] path = /usr/share/repsys/default.chlog [helper] create-srpm = /usr/share/repsys/create-srpm +upload-srpm = /usr/local/bin/youri.devel [users] -andreas = Andreas Hasenack -boiko = Gustavo Pichorim Boiko -cavassin = Wanderlei Cavassin -fcrozat = Frederic Crozat -flepied = Frederic Lepied -helio = Helio Chissini de Castro -lmontel = Laurent Montel -oden = Oden Eriksson +# jsmith = John Smith [submit] +host = kenobi.mandriva.com default = Cooker [submit Cooker] target = /export/home/repsys allowed = svn+ssh://svn.mandriva.com/svn/packages/cooker scripts = /usr/share/repsys/rebrand-mdk +## +## rpm-macros refers to the sections containing the macros used for this +## target. The values will be used to build the rpmbuild command line. For +## example: +## +## [macros cooker] +## a = b +## c = %a +## +## will render in the command line: --define "a b" --define "c %a". +## +#rpm-macros = global cooker + +#[macros global] +#distsuffix = mdv + +#[macros cooker] +#mandriva_release = 2007.1 -- cgit v1.2.1