aboutsummaryrefslogtreecommitdiffstats
path: root/MgaRepo/log.py
diff options
context:
space:
mode:
Diffstat (limited to 'MgaRepo/log.py')
-rw-r--r--MgaRepo/log.py633
1 files changed, 633 insertions, 0 deletions
diff --git a/MgaRepo/log.py b/MgaRepo/log.py
new file mode 100644
index 0000000..56965c5
--- /dev/null
+++ b/MgaRepo/log.py
@@ -0,0 +1,633 @@
+#!/usr/bin/python
+from MgaRepo import Error, config, layout
+from MgaRepo.svn import SVN
+from MgaRepo.util import execcmd
+
+try:
+ from Cheetah.Template import Template
+except ImportError:
+ raise Error, "mgarepo requires the package python-cheetah"
+
+from cStringIO import StringIO
+
+import sys
+import os
+import re
+import time
+import locale
+import glob
+import tempfile
+import shutil
+import subprocess
+
+
+locale.setlocale(locale.LC_ALL, "C")
+
+default_template = """
+#if not $releases_by_author[-1].visible
+ ## Hide the first release that contains no changes. It must be a
+ ## reimported package and the log gathered from misc/ already should
+ ## contain a correct entry for the version-release:
+ #set $releases_by_author = $releases_by_author[:-1]
+#end if
+#for $rel in $releases_by_author
+* $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
++ Revision: $rel.revision
+## #if not $rel.released
+##+ Status: not released
+## #end if
+ #if not $rel.visible
++ rebuild (emptylog)
+ #end if
+ #for $rev in $rel.release_revisions
+ #for $line in $rev.lines
+$line
+ #end for
+ #end for
+
+ #for $author in $rel.authors
+ #if not $author.visible
+ #continue
+ #end if
+ ##alternatively, one could use:
+ ###if $author.email == "root"
+ ## #continue
+ ###end if
+ + $author.name <$author.email>
+ #for $rev in $author.revisions
+ #for $line in $rev.lines
+ $line
+ #end for
+ #end for
+
+ #end for
+#end for
+"""
+
+def getrelease(pkgdirurl, rev=None, macros=[], exported=None):
+ """Tries to obtain the version-release of the package for a
+ yet-not-markrelease revision of the package.
+
+ Is here where things should be changed if "automatic release increasing"
+ will be used.
+ """
+ from MgaRepo.rpmutil import rpm_macros_defs
+ svn = SVN()
+ pkgcurrenturl = os.path.join(pkgdirurl, "current")
+ specurl = os.path.join(pkgcurrenturl, "SPECS")
+ if exported is None:
+ tmpdir = tempfile.mktemp()
+ svn.export(specurl, tmpdir, rev=rev)
+ else:
+ tmpdir = os.path.join(exported, "SPECS")
+ try:
+ found = glob.glob(os.path.join(tmpdir, "*.spec"))
+ if not found:
+ raise Error, "no .spec file found inside %s" % specurl
+ specpath = found[0]
+ options = rpm_macros_defs(macros)
+ command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' "
+ "--specfile %s %s") %
+ (specpath, options))
+ pipe = subprocess.Popen(command, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=True)
+ pipe.wait()
+ output = pipe.stdout.read()
+ error = pipe.stderr.read()
+ if pipe.returncode != 0:
+ raise Error, "Error in command %s: %s" % (command, error)
+ releases = output.split()
+ try:
+ epoch, vr = releases[0].split(":", 1)
+ version, release = vr.split("-", 1)
+ except ValueError:
+ raise Error, "Invalid command output: %s: %s" % \
+ (command, output)
+ #XXX check if this is the right way:
+ if epoch == "(none)":
+ ev = version
+ else:
+ ev = epoch + ":" + version
+ return ev, release
+ finally:
+ if exported is None and os.path.isdir(tmpdir):
+ shutil.rmtree(tmpdir)
+
+class _Revision:
+ lines = []
+ date = None
+ raw_date = None
+ revision = None
+ author_name = None
+ author_email = None
+
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+
+ def __repr__(self):
+ lines = repr(self.lines)[:30] + "...]"
+ line = "<_Revision %d author=%r date=%r lines=%s>" % \
+ (self.revision, self.author, self.date, lines)
+ return line
+
+
+class _Release(_Revision):
+ version = None
+ release = None
+ revisions = []
+ release_revisions = []
+ authors = []
+ visible = False
+
+ def __init__(self, **kwargs):
+ self.revisions = []
+ _Revision.__init__(self, **kwargs)
+
+ def __repr__(self):
+ line = "<_Release v=%s r=%s revs=%r>" % \
+ (self.version, self.release, self.revisions)
+ return line
+
+unescaped_macro_pat = re.compile(r"([^%])%([^%])")
+
+def escape_macros(text):
+ escaped = unescaped_macro_pat.sub("\\1%%\\2", text)
+ return escaped
+
+def format_lines(lines):
+ first = 1
+ entrylines = []
+ perexpr = re.compile(r"([^%])%([^%])")
+ for line in lines:
+ if line:
+ line = escape_macros(line)
+ if first:
+ first = 0
+ line = line.lstrip()
+ if line[0] != "-":
+ nextline = "- " + line
+ else:
+ nextline = line
+ elif line[0] != " " and line[0] != "-":
+ nextline = " " + line
+ else:
+ nextline = line
+ if nextline not in entrylines:
+ entrylines.append(nextline)
+ return entrylines
+
+
+class _Author:
+ name = None
+ email = None
+ revisions = None
+ visible = False
+
+
+def group_releases_by_author(releases):
+ allauthors = []
+ grouped = []
+ for release in releases:
+
+ # group revisions of the release by author
+ authors = {}
+ latest = None
+ for revision in release.revisions:
+ authors.setdefault(revision.author, []).append(revision)
+
+ # create _Authors and sort them by their latest revisions
+ decorated = []
+ for authorname, revs in authors.iteritems():
+ author = _Author()
+ author.name = revs[0].author_name
+ author.email = revs[0].author_email
+ author.revisions = revs
+ # #41117: mark those authors without visible messages
+ author.visible = bool(sum(len(rev.lines) for rev in revs))
+ revlatest = author.revisions[0]
+ # keep the latest revision even for completely invisible
+ # authors (below)
+ if latest is None or revlatest.revision > latest.revision:
+ latest = revlatest
+ if not author.visible:
+ # only sort those visible authors, invisible ones are used
+ # only in "latest"
+ continue
+ decorated.append((revlatest.revision, author))
+ decorated.sort(reverse=1)
+
+ if release.visible:
+ release.authors = [t[1] for t in decorated]
+ firstrel, release.authors = release.authors[0], release.authors[1:]
+ release.author_name = firstrel.name
+ release.author_email = firstrel.email
+ release.release_revisions = firstrel.revisions
+ else:
+ # we don't care about other possible authors in completely
+ # invisible releases
+ firstrev = release.revisions[0]
+ release.author_name = firstrev.author_name
+ release.author_email = firstrev.author_email
+ release.raw_date = firstrev.raw_date
+ release.date = firstrev.date
+
+ release.date = latest.date
+ release.raw_date = latest.raw_date
+ release.revision = latest.revision
+
+ grouped.append(release)
+
+ return grouped
+
+
+def group_revisions_by_author(currentlog):
+ revisions = []
+ last_author = None
+ for entry in currentlog:
+ revision = _Revision()
+ revision.lines = format_lines(entry.lines)
+ revision.raw_date = entry.date
+ revision.date = parse_raw_date(entry.date)
+ revision.revision = entry.revision
+ if entry.author == last_author:
+ revisions[-1].revisions.append(revision)
+ else:
+ author = _Author()
+ author.name, author.email = get_author_name(entry.author)
+ author.revisions = [revision]
+ revisions.append(author)
+ last_author = entry.author
+ return revisions
+
+
+emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>")
+
+def get_author_name(author):
+ found = emailpat.match(config.get("users", author, author))
+ name = ((found and found.group("name")) or author)
+ email = ((found and found.group("email")) or author)
+ return name, email
+
+def parse_raw_date(rawdate):
+ return time.strftime("%a %b %d %Y", rawdate)
+
+def filter_log_lines(lines):
+ # Lines in commit messages beginning with CLOG will be the only shown
+ # in the changelog. These lines will have the CLOG token and blanks
+ # stripped from the beginning.
+ onlylines = None
+ clogstr = config.get("log", "unignore-string")
+ if clogstr:
+ clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr)
+ onlylines = [clogre.sub("", line)
+ for line in lines if line.startswith(clogstr)]
+ if onlylines:
+ filtered = onlylines
+ else:
+ # Lines in commit messages containing SILENT at any position will be
+ # skipped; commits with their log messages beggining with SILENT in the
+ # first positionj of the first line will have all lines ignored.
+ ignstr = config.get("log", "ignore-string", "SILENT")
+ if len(lines) and lines[0].startswith(ignstr):
+ return []
+ filtered = [line for line in lines if ignstr not in line]
+ return filtered
+
+
+def make_release(author=None, revision=None, date=None, lines=None,
+ entries=[], released=True, version=None, release=None):
+ rel = _Release()
+ rel.author = author
+ if author:
+ rel.author_name, rel.author_email = get_author_name(author)
+ rel.revision = revision
+ rel.version = version
+ rel.release = release
+ rel.date = (date and parse_raw_date(date)) or None
+ rel.lines = lines
+ rel.released = released
+ rel.visible = False
+ for entry in entries:
+ lines = filter_log_lines(entry.lines)
+ revision = _Revision()
+ revision.revision = entry.revision
+ revision.lines = format_lines(lines)
+ if revision.lines:
+ rel.visible = True
+ revision.date = parse_raw_date(entry.date)
+ revision.raw_date = entry.date
+ revision.author = entry.author
+ (revision.author_name, revision.author_email) = \
+ get_author_name(entry.author)
+ rel.revisions.append(revision)
+ return rel
+
+
+def dump_file(releases, currentlog=None, template=None):
+ templpath = template or config.get("template", "path",
+ "/usr/share/mgarepo/default.chlog")
+ params = {}
+ if templpath is None or not os.path.exists(templpath):
+ params["source"] = default_template
+ sys.stderr.write("warning: %s not found. using built-in template.\n"%
+ templpath)
+ else:
+ params["file"] = templpath
+ releases_author = group_releases_by_author(releases)
+ revisions_author = group_revisions_by_author(currentlog)
+ params["searchList"] = [{"releases_by_author" : releases_author,
+ "releases" : releases,
+ "revisions_by_author": revisions_author}]
+ t = Template(**params)
+ return t.respond()
+
+
+class InvalidEntryError(Exception):
+ pass
+
+def parse_repsys_entry(revlog):
+ # parse entries in the format:
+ # %repsys <operation>
+ # key: value
+ # ..
+ # <newline>
+ # <comments>
+ #
+ if len(revlog.lines) == 0 or not revlog.lines[0].startswith("%repsys"):
+ raise InvalidEntryError
+ try:
+ data = {"operation" : revlog.lines[0].split()[1]}
+ except IndexError:
+ raise InvalidEntryError
+ for line in revlog.lines[1:]:
+ if not line:
+ break
+ try:
+ key, value = line.split(":", 1)
+ except ValueError:
+ raise InvalidEntryError
+ data[key.strip().lower()] = value.strip() # ???
+ return data
+
+
+def get_revision_offset():
+ try:
+ revoffset = config.getint("log", "revision-offset", 0)
+ except (ValueError, TypeError):
+ raise Error, ("Invalid revision-offset number in configuration "
+ "file(s).")
+ return revoffset or 0
+
+oldmsgpat = re.compile(
+ r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.")
+
+def parse_markrelease_log(relentry):
+ if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \
+ or parse_repsys_entry(relentry))):
+ raise InvalidEntryError
+ from_rev = None
+ path = None
+ for changed in relentry.changed:
+ if changed["action"] == "A" and changed["from_rev"]:
+ from_rev = changed["from_rev"]
+ path = changed["path"]
+ break
+ else:
+ raise InvalidEntryError
+ # get the version and release from the names in the path, do not relay
+ # on log messages
+ version, release = path.rsplit(os.path.sep, 3)[-2:]
+ return version, release, from_rev
+
+
+def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
+ template=None, macros=[], exported=None):
+ concat = config.get("log", "concat", "").split()
+ revoffset = get_revision_offset()
+ svn = SVN()
+ pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True)
+ pkgcurrenturl = layout.checkout_url(pkgdirurl)
+ releaseslog = svn.log(pkgreleasesurl, noerror=1)
+ currentlog = svn.log(pkgcurrenturl, limit=size, start=rev,
+ end=revoffset)
+
+ # sort releases by copyfrom-revision, so that markreleases for same
+ # revisions won't look empty
+ releasesdata = []
+ if releaseslog:
+ for relentry in releaseslog[::-1]:
+ try:
+ (version, release, relrevision) = \
+ parse_markrelease_log(relentry)
+ except InvalidEntryError:
+ continue
+ releasesdata.append((relrevision, -relentry.revision, relentry,
+ version, release))
+ releasesdata.sort()
+
+ # collect valid releases using the versions provided by the changes and
+ # the packages
+ prevrevision = 0
+ releases = []
+ for (relrevision, dummy, relentry, version, release) in releasesdata:
+ if prevrevision == relrevision:
+ # ignore older markrelease of the same revision, since they
+ # will have no history
+ continue
+ entries = [entry for entry in currentlog
+ if relrevision >= entry.revision and
+ (prevrevision < entry.revision)]
+ if not entries:
+ #XXX probably a forced release, without commits in current/,
+ # check if this is the right behavior
+ sys.stderr.write("warning: skipping (possible) release "
+ "%s-%s@%s, no commits since previous markrelease (r%r)\n" %
+ (version, release, relrevision, prevrevision))
+ continue
+
+ release = make_release(author=relentry.author,
+ revision=relentry.revision, date=relentry.date,
+ lines=relentry.lines, entries=entries,
+ version=version, release=release)
+ releases.append(release)
+ prevrevision = relrevision
+
+ # look for commits that have been not submitted (released) yet
+ # this is done by getting all log entries newer (greater revision no.)
+ # than releasesdata[-1] (in the case it exists)
+ if releasesdata:
+ latest_revision = releasesdata[-1][0] # the latest copied rev
+ else:
+ latest_revision = 0
+ notsubmitted = [entry for entry in currentlog
+ if entry.revision > latest_revision]
+ if notsubmitted:
+ # if they are not submitted yet, what we have to do is to add
+ # a release/version number from getrelease()
+ version, release = getrelease(pkgdirurl, macros=macros,
+ exported=exported)
+ toprelease = make_release(entries=notsubmitted, released=False,
+ version=version, release=release)
+ releases.append(toprelease)
+
+ data = dump_file(releases[::-1], currentlog=currentlog, template=template)
+ return data
+
+def _split_changelog(stream):
+ current = None
+ count = 0
+ def finish(entry):
+ lines = entry[2]
+ # strip newlines at the end
+ for i in xrange(len(lines)-1, -1, -1):
+ if lines[i] != "\n":
+ break
+ del lines[i]
+ return entry
+ for line in stream:
+ if line.startswith("*"):
+ if current:
+ yield finish(current)
+ fields = line.split()
+ rawdate = " ".join(fields[:5])
+ try:
+ date = time.strptime(rawdate, "* %a %b %d %Y")
+ except ValueError, e:
+ raise Error, "failed to parse spec changelog: %s" % e
+ curlines = [line]
+ current = (date, count, curlines)
+ # count used to ensure stable sorting when changelog entries
+ # have the same date, otherwise it would also compare the
+ # changelog lines
+ count -= 1
+ elif current:
+ curlines.append(line)
+ else:
+ pass # not good, but ignore
+ if current:
+ yield finish(current)
+
+def sort_changelog(stream):
+ entries = _split_changelog(stream)
+ log = StringIO()
+ for time, count, elines in sorted(entries, reverse=True):
+ log.writelines(elines)
+ log.write("\n")
+ return log
+
+def split_spec_changelog(stream):
+ chlog = StringIO()
+ spec = StringIO()
+ found = 0
+ visible = 0
+ for line in stream:
+ if line.startswith("%changelog"):
+ found = 1
+ elif not found:
+ spec.write(line)
+ elif found:
+ if line.strip():
+ visible = 1
+ chlog.write(line)
+ elif line.startswith("%"):
+ found = 0
+ spec.write(line)
+ spec.seek(0)
+ if not visible:
+ # when there are only blanks in the changelog, make it empty
+ chlog = StringIO()
+ return spec, chlog
+
+def get_old_log(pkgdirurl):
+ chlog = StringIO()
+ oldurl = config.get("log", "oldurl")
+ if oldurl:
+ svn = SVN()
+ tmpdir = tempfile.mktemp()
+ try:
+ pkgname = layout.package_name(pkgdirurl)
+ pkgoldurl = os.path.join(oldurl, pkgname)
+ try:
+ # we're using HEAD here because fixes in misc/ (oldurl) may
+ # be newer than packages' last changed revision.
+ svn.export(pkgoldurl, tmpdir)
+ except Error:
+ pass
+ else:
+ logfile = os.path.join(tmpdir, "log")
+ if os.path.isfile(logfile):
+ file = open(logfile)
+ chlog.write("\n") # TODO needed?
+ log = file.read()
+ log = escape_macros(log)
+ chlog.write(log)
+ file.close()
+ finally:
+ if os.path.isdir(tmpdir):
+ shutil.rmtree(tmpdir)
+ chlog.seek(0)
+ return chlog
+
+def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
+ submit=False, sort=False, template=None, macros=[], exported=None,
+ oldlog=False):
+ """Generates the changelog for a given package URL
+
+ @another: a stream with the contents of a changelog to be merged with
+ the one generated
+ @svn: enable changelog from svn
+ @rev: generate the changelog with the changes up to the given
+ revision
+ @size: the number of revisions to be used (as in svn log --limit)
+ @submit: defines whether the latest unreleased log entries should have
+ the version parsed from the spec file
+ @sort: should changelog entries be reparsed and sorted after appending
+ the oldlog?
+ @template: the path to the cheetah template used to generate the
+ changelog from svn
+ @macros: a list of tuples containing macros to be defined when
+ parsing the version in the changelog
+ @exported: the path of a directory containing an already existing
+ checkout of the package, so that the spec file can be
+ parsed from there
+ @oldlog: if set it will try to append the old changelog file defined
+ in oldurl in mgarepo.conf
+ """
+ newlog = StringIO()
+ if svn:
+ rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
+ template=template, macros=macros, exported=exported)
+ newlog.write(rawsvnlog)
+ if another:
+ newlog.writelines(another)
+ if oldlog:
+ newlog.writelines(get_old_log(pkgdirurl))
+ if sort:
+ newlog.seek(0)
+ newlog = sort_changelog(newlog)
+ newlog.seek(0)
+ return newlog
+
+def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
+ submit=False, sort=False, template=None, macros=[], exported=None):
+ fi = open(specfile)
+ spec, oldchlog = split_spec_changelog(fi)
+ fi.close()
+ another = None
+ if config.getbool("log", "merge-spec", False):
+ another = oldchlog
+ sort = sort or config.getbool("log", "sort", False)
+ chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size,
+ submit=submit, sort=sort, template=template, macros=macros,
+ exported=exported, oldlog=True)
+ fo = open(specfile, "w")
+ fo.writelines(spec)
+ fo.write("\n\n%changelog\n")
+ fo.writelines(chlog)
+ fo.close()
+
+if __name__ == "__main__":
+ l = svn2rpm(sys.argv[1])
+ print l
+
+# vim:et:ts=4:sw=4