diff options
Diffstat (limited to 'MgaRepo/log.py')
| -rw-r--r-- | MgaRepo/log.py | 633 |
1 files changed, 633 insertions, 0 deletions
diff --git a/MgaRepo/log.py b/MgaRepo/log.py new file mode 100644 index 0000000..56965c5 --- /dev/null +++ b/MgaRepo/log.py @@ -0,0 +1,633 @@ +#!/usr/bin/python +from MgaRepo import Error, config, layout +from MgaRepo.svn import SVN +from MgaRepo.util import execcmd + +try: + from Cheetah.Template import Template +except ImportError: + raise Error, "mgarepo requires the package python-cheetah" + +from cStringIO import StringIO + +import sys +import os +import re +import time +import locale +import glob +import tempfile +import shutil +import subprocess + + +locale.setlocale(locale.LC_ALL, "C") + +default_template = """ +#if not $releases_by_author[-1].visible + ## Hide the first release that contains no changes. It must be a + ## reimported package and the log gathered from misc/ already should + ## contain a correct entry for the version-release: + #set $releases_by_author = $releases_by_author[:-1] +#end if +#for $rel in $releases_by_author +* $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release ++ Revision: $rel.revision +## #if not $rel.released +##+ Status: not released +## #end if + #if not $rel.visible ++ rebuild (emptylog) + #end if + #for $rev in $rel.release_revisions + #for $line in $rev.lines +$line + #end for + #end for + + #for $author in $rel.authors + #if not $author.visible + #continue + #end if + ##alternatively, one could use: + ###if $author.email == "root" + ## #continue + ###end if + + $author.name <$author.email> + #for $rev in $author.revisions + #for $line in $rev.lines + $line + #end for + #end for + + #end for +#end for +""" + +def getrelease(pkgdirurl, rev=None, macros=[], exported=None): + """Tries to obtain the version-release of the package for a + yet-not-markrelease revision of the package. + + Is here where things should be changed if "automatic release increasing" + will be used. + """ + from MgaRepo.rpmutil import rpm_macros_defs + svn = SVN() + pkgcurrenturl = os.path.join(pkgdirurl, "current") + specurl = os.path.join(pkgcurrenturl, "SPECS") + if exported is None: + tmpdir = tempfile.mktemp() + svn.export(specurl, tmpdir, rev=rev) + else: + tmpdir = os.path.join(exported, "SPECS") + try: + found = glob.glob(os.path.join(tmpdir, "*.spec")) + if not found: + raise Error, "no .spec file found inside %s" % specurl + specpath = found[0] + options = rpm_macros_defs(macros) + command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " + "--specfile %s %s") % + (specpath, options)) + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + pipe.wait() + output = pipe.stdout.read() + error = pipe.stderr.read() + if pipe.returncode != 0: + raise Error, "Error in command %s: %s" % (command, error) + releases = output.split() + try: + epoch, vr = releases[0].split(":", 1) + version, release = vr.split("-", 1) + except ValueError: + raise Error, "Invalid command output: %s: %s" % \ + (command, output) + #XXX check if this is the right way: + if epoch == "(none)": + ev = version + else: + ev = epoch + ":" + version + return ev, release + finally: + if exported is None and os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + +class _Revision: + lines = [] + date = None + raw_date = None + revision = None + author_name = None + author_email = None + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __repr__(self): + lines = repr(self.lines)[:30] + "...]" + line = "<_Revision %d author=%r date=%r lines=%s>" % \ + (self.revision, self.author, self.date, lines) + return line + + +class _Release(_Revision): + version = None + release = None + revisions = [] + release_revisions = [] + authors = [] + visible = False + + def __init__(self, **kwargs): + self.revisions = [] + _Revision.__init__(self, **kwargs) + + def __repr__(self): + line = "<_Release v=%s r=%s revs=%r>" % \ + (self.version, self.release, self.revisions) + return line + +unescaped_macro_pat = re.compile(r"([^%])%([^%])") + +def escape_macros(text): + escaped = unescaped_macro_pat.sub("\\1%%\\2", text) + return escaped + +def format_lines(lines): + first = 1 + entrylines = [] + perexpr = re.compile(r"([^%])%([^%])") + for line in lines: + if line: + line = escape_macros(line) + if first: + first = 0 + line = line.lstrip() + if line[0] != "-": + nextline = "- " + line + else: + nextline = line + elif line[0] != " " and line[0] != "-": + nextline = " " + line + else: + nextline = line + if nextline not in entrylines: + entrylines.append(nextline) + return entrylines + + +class _Author: + name = None + email = None + revisions = None + visible = False + + +def group_releases_by_author(releases): + allauthors = [] + grouped = [] + for release in releases: + + # group revisions of the release by author + authors = {} + latest = None + for revision in release.revisions: + authors.setdefault(revision.author, []).append(revision) + + # create _Authors and sort them by their latest revisions + decorated = [] + for authorname, revs in authors.iteritems(): + author = _Author() + author.name = revs[0].author_name + author.email = revs[0].author_email + author.revisions = revs + # #41117: mark those authors without visible messages + author.visible = bool(sum(len(rev.lines) for rev in revs)) + revlatest = author.revisions[0] + # keep the latest revision even for completely invisible + # authors (below) + if latest is None or revlatest.revision > latest.revision: + latest = revlatest + if not author.visible: + # only sort those visible authors, invisible ones are used + # only in "latest" + continue + decorated.append((revlatest.revision, author)) + decorated.sort(reverse=1) + + if release.visible: + release.authors = [t[1] for t in decorated] + firstrel, release.authors = release.authors[0], release.authors[1:] + release.author_name = firstrel.name + release.author_email = firstrel.email + release.release_revisions = firstrel.revisions + else: + # we don't care about other possible authors in completely + # invisible releases + firstrev = release.revisions[0] + release.author_name = firstrev.author_name + release.author_email = firstrev.author_email + release.raw_date = firstrev.raw_date + release.date = firstrev.date + + release.date = latest.date + release.raw_date = latest.raw_date + release.revision = latest.revision + + grouped.append(release) + + return grouped + + +def group_revisions_by_author(currentlog): + revisions = [] + last_author = None + for entry in currentlog: + revision = _Revision() + revision.lines = format_lines(entry.lines) + revision.raw_date = entry.date + revision.date = parse_raw_date(entry.date) + revision.revision = entry.revision + if entry.author == last_author: + revisions[-1].revisions.append(revision) + else: + author = _Author() + author.name, author.email = get_author_name(entry.author) + author.revisions = [revision] + revisions.append(author) + last_author = entry.author + return revisions + + +emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>") + +def get_author_name(author): + found = emailpat.match(config.get("users", author, author)) + name = ((found and found.group("name")) or author) + email = ((found and found.group("email")) or author) + return name, email + +def parse_raw_date(rawdate): + return time.strftime("%a %b %d %Y", rawdate) + +def filter_log_lines(lines): + # Lines in commit messages beginning with CLOG will be the only shown + # in the changelog. These lines will have the CLOG token and blanks + # stripped from the beginning. + onlylines = None + clogstr = config.get("log", "unignore-string") + if clogstr: + clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr) + onlylines = [clogre.sub("", line) + for line in lines if line.startswith(clogstr)] + if onlylines: + filtered = onlylines + else: + # Lines in commit messages containing SILENT at any position will be + # skipped; commits with their log messages beggining with SILENT in the + # first positionj of the first line will have all lines ignored. + ignstr = config.get("log", "ignore-string", "SILENT") + if len(lines) and lines[0].startswith(ignstr): + return [] + filtered = [line for line in lines if ignstr not in line] + return filtered + + +def make_release(author=None, revision=None, date=None, lines=None, + entries=[], released=True, version=None, release=None): + rel = _Release() + rel.author = author + if author: + rel.author_name, rel.author_email = get_author_name(author) + rel.revision = revision + rel.version = version + rel.release = release + rel.date = (date and parse_raw_date(date)) or None + rel.lines = lines + rel.released = released + rel.visible = False + for entry in entries: + lines = filter_log_lines(entry.lines) + revision = _Revision() + revision.revision = entry.revision + revision.lines = format_lines(lines) + if revision.lines: + rel.visible = True + revision.date = parse_raw_date(entry.date) + revision.raw_date = entry.date + revision.author = entry.author + (revision.author_name, revision.author_email) = \ + get_author_name(entry.author) + rel.revisions.append(revision) + return rel + + +def dump_file(releases, currentlog=None, template=None): + templpath = template or config.get("template", "path", + "/usr/share/mgarepo/default.chlog") + params = {} + if templpath is None or not os.path.exists(templpath): + params["source"] = default_template + sys.stderr.write("warning: %s not found. using built-in template.\n"% + templpath) + else: + params["file"] = templpath + releases_author = group_releases_by_author(releases) + revisions_author = group_revisions_by_author(currentlog) + params["searchList"] = [{"releases_by_author" : releases_author, + "releases" : releases, + "revisions_by_author": revisions_author}] + t = Template(**params) + return t.respond() + + +class InvalidEntryError(Exception): + pass + +def parse_repsys_entry(revlog): + # parse entries in the format: + # %repsys <operation> + # key: value + # .. + # <newline> + # <comments> + # + if len(revlog.lines) == 0 or not revlog.lines[0].startswith("%repsys"): + raise InvalidEntryError + try: + data = {"operation" : revlog.lines[0].split()[1]} + except IndexError: + raise InvalidEntryError + for line in revlog.lines[1:]: + if not line: + break + try: + key, value = line.split(":", 1) + except ValueError: + raise InvalidEntryError + data[key.strip().lower()] = value.strip() # ??? + return data + + +def get_revision_offset(): + try: + revoffset = config.getint("log", "revision-offset", 0) + except (ValueError, TypeError): + raise Error, ("Invalid revision-offset number in configuration " + "file(s).") + return revoffset or 0 + +oldmsgpat = re.compile( + r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.") + +def parse_markrelease_log(relentry): + if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \ + or parse_repsys_entry(relentry))): + raise InvalidEntryError + from_rev = None + path = None + for changed in relentry.changed: + if changed["action"] == "A" and changed["from_rev"]: + from_rev = changed["from_rev"] + path = changed["path"] + break + else: + raise InvalidEntryError + # get the version and release from the names in the path, do not relay + # on log messages + version, release = path.rsplit(os.path.sep, 3)[-2:] + return version, release, from_rev + + +def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, + template=None, macros=[], exported=None): + concat = config.get("log", "concat", "").split() + revoffset = get_revision_offset() + svn = SVN() + pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True) + pkgcurrenturl = layout.checkout_url(pkgdirurl) + releaseslog = svn.log(pkgreleasesurl, noerror=1) + currentlog = svn.log(pkgcurrenturl, limit=size, start=rev, + end=revoffset) + + # sort releases by copyfrom-revision, so that markreleases for same + # revisions won't look empty + releasesdata = [] + if releaseslog: + for relentry in releaseslog[::-1]: + try: + (version, release, relrevision) = \ + parse_markrelease_log(relentry) + except InvalidEntryError: + continue + releasesdata.append((relrevision, -relentry.revision, relentry, + version, release)) + releasesdata.sort() + + # collect valid releases using the versions provided by the changes and + # the packages + prevrevision = 0 + releases = [] + for (relrevision, dummy, relentry, version, release) in releasesdata: + if prevrevision == relrevision: + # ignore older markrelease of the same revision, since they + # will have no history + continue + entries = [entry for entry in currentlog + if relrevision >= entry.revision and + (prevrevision < entry.revision)] + if not entries: + #XXX probably a forced release, without commits in current/, + # check if this is the right behavior + sys.stderr.write("warning: skipping (possible) release " + "%s-%s@%s, no commits since previous markrelease (r%r)\n" % + (version, release, relrevision, prevrevision)) + continue + + release = make_release(author=relentry.author, + revision=relentry.revision, date=relentry.date, + lines=relentry.lines, entries=entries, + version=version, release=release) + releases.append(release) + prevrevision = relrevision + + # look for commits that have been not submitted (released) yet + # this is done by getting all log entries newer (greater revision no.) + # than releasesdata[-1] (in the case it exists) + if releasesdata: + latest_revision = releasesdata[-1][0] # the latest copied rev + else: + latest_revision = 0 + notsubmitted = [entry for entry in currentlog + if entry.revision > latest_revision] + if notsubmitted: + # if they are not submitted yet, what we have to do is to add + # a release/version number from getrelease() + version, release = getrelease(pkgdirurl, macros=macros, + exported=exported) + toprelease = make_release(entries=notsubmitted, released=False, + version=version, release=release) + releases.append(toprelease) + + data = dump_file(releases[::-1], currentlog=currentlog, template=template) + return data + +def _split_changelog(stream): + current = None + count = 0 + def finish(entry): + lines = entry[2] + # strip newlines at the end + for i in xrange(len(lines)-1, -1, -1): + if lines[i] != "\n": + break + del lines[i] + return entry + for line in stream: + if line.startswith("*"): + if current: + yield finish(current) + fields = line.split() + rawdate = " ".join(fields[:5]) + try: + date = time.strptime(rawdate, "* %a %b %d %Y") + except ValueError, e: + raise Error, "failed to parse spec changelog: %s" % e + curlines = [line] + current = (date, count, curlines) + # count used to ensure stable sorting when changelog entries + # have the same date, otherwise it would also compare the + # changelog lines + count -= 1 + elif current: + curlines.append(line) + else: + pass # not good, but ignore + if current: + yield finish(current) + +def sort_changelog(stream): + entries = _split_changelog(stream) + log = StringIO() + for time, count, elines in sorted(entries, reverse=True): + log.writelines(elines) + log.write("\n") + return log + +def split_spec_changelog(stream): + chlog = StringIO() + spec = StringIO() + found = 0 + visible = 0 + for line in stream: + if line.startswith("%changelog"): + found = 1 + elif not found: + spec.write(line) + elif found: + if line.strip(): + visible = 1 + chlog.write(line) + elif line.startswith("%"): + found = 0 + spec.write(line) + spec.seek(0) + if not visible: + # when there are only blanks in the changelog, make it empty + chlog = StringIO() + return spec, chlog + +def get_old_log(pkgdirurl): + chlog = StringIO() + oldurl = config.get("log", "oldurl") + if oldurl: + svn = SVN() + tmpdir = tempfile.mktemp() + try: + pkgname = layout.package_name(pkgdirurl) + pkgoldurl = os.path.join(oldurl, pkgname) + try: + # we're using HEAD here because fixes in misc/ (oldurl) may + # be newer than packages' last changed revision. + svn.export(pkgoldurl, tmpdir) + except Error: + pass + else: + logfile = os.path.join(tmpdir, "log") + if os.path.isfile(logfile): + file = open(logfile) + chlog.write("\n") # TODO needed? + log = file.read() + log = escape_macros(log) + chlog.write(log) + file.close() + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) + chlog.seek(0) + return chlog + +def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, + submit=False, sort=False, template=None, macros=[], exported=None, + oldlog=False): + """Generates the changelog for a given package URL + + @another: a stream with the contents of a changelog to be merged with + the one generated + @svn: enable changelog from svn + @rev: generate the changelog with the changes up to the given + revision + @size: the number of revisions to be used (as in svn log --limit) + @submit: defines whether the latest unreleased log entries should have + the version parsed from the spec file + @sort: should changelog entries be reparsed and sorted after appending + the oldlog? + @template: the path to the cheetah template used to generate the + changelog from svn + @macros: a list of tuples containing macros to be defined when + parsing the version in the changelog + @exported: the path of a directory containing an already existing + checkout of the package, so that the spec file can be + parsed from there + @oldlog: if set it will try to append the old changelog file defined + in oldurl in mgarepo.conf + """ + newlog = StringIO() + if svn: + rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, + template=template, macros=macros, exported=exported) + newlog.write(rawsvnlog) + if another: + newlog.writelines(another) + if oldlog: + newlog.writelines(get_old_log(pkgdirurl)) + if sort: + newlog.seek(0) + newlog = sort_changelog(newlog) + newlog.seek(0) + return newlog + +def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, + submit=False, sort=False, template=None, macros=[], exported=None): + fi = open(specfile) + spec, oldchlog = split_spec_changelog(fi) + fi.close() + another = None + if config.getbool("log", "merge-spec", False): + another = oldchlog + sort = sort or config.getbool("log", "sort", False) + chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size, + submit=submit, sort=sort, template=template, macros=macros, + exported=exported, oldlog=True) + fo = open(specfile, "w") + fo.writelines(spec) + fo.write("\n\n%changelog\n") + fo.writelines(chlog) + fo.close() + +if __name__ == "__main__": + l = svn2rpm(sys.argv[1]) + print l + +# vim:et:ts=4:sw=4 |