path: root/MgaRepo/log.py

#!/usr/bin/python
from MgaRepo import Error, config, layout
from MgaRepo.svn import SVN
from MgaRepo.util import execcmd

try:
    from Cheetah.Template import Template
except ImportError:
    raise Error, "mgarepo requires the package python-cheetah"

from cStringIO import StringIO

import sys
import os
import re
import time
import locale
import glob
import tempfile
import shutil
import subprocess


locale.setlocale(locale.LC_ALL, "C")

default_template = """
#if not $releases_by_author[-1].visible
  ## Hide the first release that contains no changes. It must be a
  ## reimported package and the log gathered from misc/ already should
  ## contain a correct entry for the version-release:
  #set $releases_by_author = $releases_by_author[:-1]
#end if
#for $rel in $releases_by_author
* $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
+ Revision: $rel.revision
## #if not $rel.released
##+ Status: not released
## #end if
 #if not $rel.visible
+ rebuild (emptylog)
 #end if
 #for $rev in $rel.release_revisions
  #for $line in $rev.lines
$line
  #end for
 #end for

 #for $author in $rel.authors
  #if not $author.visible
    #continue
  #end if
  ##alternatively, one could use:
  ###if $author.email == "root"
  ## #continue
  ###end if
  + $author.name <$author.email>
  #for $rev in $author.revisions
    #for $line in $rev.lines
    $line
    #end for
  #end for

 #end for
#end for
"""

def getrelease(pkgdirurl, rev=None, macros=[], exported=None):
    """Tries to obtain the version-release of the package for a 
    yet-not-markrelease revision of the package.

    Is here where things should be changed if "automatic release increasing" 
    will be used.
    """
    from MgaRepo.rpmutil import rpm_macros_defs
    svn = SVN()
    pkgcurrenturl = os.path.join(pkgdirurl, "current")
    specurl = os.path.join(pkgcurrenturl, "SPECS")
    if exported is None:
        tmpdir = tempfile.mktemp()
        svn.export(specurl, tmpdir, rev=rev)
    else:
        tmpdir = os.path.join(exported, "SPECS")
    try:
        found = glob.glob(os.path.join(tmpdir, "*.spec"))
        if not found:
            raise Error, "no .spec file found inside %s" % specurl
        specpath = found[0]
        options = rpm_macros_defs(macros)
        command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' "
                   "--specfile %s %s") %
                   (specpath, options))
        pipe = subprocess.Popen(command, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE, shell=True)
        pipe.wait()
        output = pipe.stdout.read()
        error = pipe.stderr.read()
        if pipe.returncode != 0:
            raise Error, "Error in command %s: %s" % (command, error)
        releases = output.split()
        try:
            epoch, vr = releases[0].split(":", 1)
            version, release = vr.split("-", 1)
        except ValueError:
            raise Error, "Invalid command output: %s: %s" % \
                    (command, output)
        #XXX check if this is the right way:
        if epoch == "(none)":
            ev = version
        else:
            ev = epoch + ":" + version
        return ev, release
    finally:
        if exported is None and os.path.isdir(tmpdir):
            shutil.rmtree(tmpdir)
            
class _Revision:
    lines = []
    date = None
    raw_date = None
    revision = None
    author_name = None
    author_email = None

    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def __repr__(self):
        lines = repr(self.lines)[:30] + "...]" 
        line = "<_Revision %d author=%r date=%r lines=%s>" % \
                    (self.revision, self.author, self.date, lines)
        return line


class _Release(_Revision):
    version = None
    release = None
    revisions = []
    release_revisions = []
    authors = []
    visible = False

    def __init__(self, **kwargs):
        self.revisions = []
        _Revision.__init__(self, **kwargs)

    def __repr__(self):
        line = "<_Release v=%s r=%s revs=%r>" % \
                    (self.version, self.release, self.revisions)
        return line

unescaped_macro_pat = re.compile(r"(^|[^%])%([^%])")

def escape_macros(text):
    escaped = unescaped_macro_pat.sub("\\1%%\\2", text)
    return escaped

def format_lines(lines):
    first = 1
    entrylines = []
    perexpr = re.compile(r"([^%])%([^%])")
    for line in lines:
        if line:
            line = escape_macros(line)
            if first:
                first = 0
                line = line.lstrip()
                if line[0] != "-":
                    nextline = "- " + line
                else:
                    nextline = line
            elif line[0] != " " and line[0] != "-":
                nextline = "  " + line
            else:
                nextline = line
            if nextline not in entrylines:
                entrylines.append(nextline)
    return entrylines


class _Author:
    name = None
    email = None
    revisions = None
    visible = False


def group_releases_by_author(releases):
    allauthors = []
    grouped = []
    for release in releases:

        # group revisions of the release by author
        authors = {}
        latest = None
        for revision in release.revisions:
            authors.setdefault(revision.author, []).append(revision)

        # create _Authors and sort them by their latest revisions
        decorated = []
        for authorname, revs in authors.iteritems():
            author = _Author()
            author.name = revs[0].author_name
            author.email = revs[0].author_email
            author.revisions = revs
            # #41117: mark those authors without visible messages
            author.visible = bool(sum(len(rev.lines) for rev in revs))
            revlatest = author.revisions[0]
            # keep the latest revision even for completely invisible
            # authors (below)
            if latest is None or revlatest.revision > latest.revision:
                latest = revlatest
            if not author.visible:
                # only sort those visible authors, invisible ones are used
                # only in "latest"
                continue
            decorated.append((revlatest.revision, author))
        decorated.sort(reverse=1)

        if release.visible:
            release.authors = [t[1] for t in decorated]
            firstrel, release.authors = release.authors[0], release.authors[1:]
            release.author_name = firstrel.name
            release.author_email = firstrel.email
            release.release_revisions = firstrel.revisions
        else:
            # we don't care about other possible authors in completely
            # invisible releases
            firstrev = release.revisions[0]
            release.author_name = firstrev.author_name
            release.author_email = firstrev.author_email
            release.raw_date = firstrev.raw_date
            release.date = firstrev.date

        release.date = latest.date
        release.raw_date = latest.raw_date
        release.revision = latest.revision

        grouped.append(release)

    return grouped


def group_revisions_by_author(currentlog):
    revisions = []
    last_author = None
    for entry in currentlog:
        revision = _Revision()
        revision.lines = format_lines(entry.lines)
        revision.raw_date = entry.date
        revision.date = parse_raw_date(entry.date)
        revision.revision = entry.revision
        if entry.author == last_author:
            revisions[-1].revisions.append(revision)
        else:
            author = _Author()
            author.name, author.email = get_author_name(entry.author)
            author.revisions = [revision]
            revisions.append(author)
        last_author = entry.author
    return revisions


emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>")

def get_author_name(author):
    found = emailpat.match(config.get("users", author, author))
    name = ((found and found.group("name")) or author)
    email = ((found and found.group("email")) or author)
    return name, email

def parse_raw_date(rawdate):
    return time.strftime("%a %b %d %Y", rawdate)

def filter_log_lines(lines):
    # Lines in commit messages beginning with CLOG will be the only shown
    # in the changelog. These lines will have the CLOG token and blanks
    # stripped from the beginning.
    onlylines = None
    clogstr = config.get("log", "unignore-string")
    if clogstr:
        clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr)
        onlylines = [clogre.sub("", line)
                for line in lines if line.startswith(clogstr)]
    if onlylines:
        filtered = onlylines
    else:
        # Lines in commit messages containing SILENT at any position will be
        # skipped; commits with their log messages beggining with SILENT in the
        # first positionj of the first line will have all lines ignored.
        ignstr = config.get("log", "ignore-string", "SILENT")
        if len(lines) and lines[0].startswith(ignstr):
            return []
        filtered = [line for line in lines if ignstr not in line]
    return filtered


def make_release(author=None, revision=None, date=None, lines=None,
        entries=[], released=True, version=None, release=None):
    rel = _Release()
    rel.author = author
    if author:
        rel.author_name, rel.author_email = get_author_name(author)
    rel.revision = revision
    rel.version = version
    rel.release = release
    rel.date = (date and parse_raw_date(date)) or None
    rel.lines = lines
    rel.released = released
    rel.visible = False
    for entry in entries:
        lines = filter_log_lines(entry.lines)
        revision = _Revision()
        revision.revision = entry.revision
        revision.lines = format_lines(lines)
        if revision.lines:
            rel.visible = True
        revision.date = parse_raw_date(entry.date)
        revision.raw_date = entry.date
        revision.author = entry.author
        (revision.author_name, revision.author_email) = \
                get_author_name(entry.author)
        rel.revisions.append(revision)
    return rel


def dump_file(releases, currentlog=None, template=None):
    templpath = template or config.get("template", "path",
            "/usr/share/mgarepo/default.chlog")
    params = {}
    if templpath is None or not os.path.exists(templpath):
        params["source"] = default_template
        sys.stderr.write("warning: %s not found. using built-in template.\n"%
                templpath)
    else:
        params["file"] = templpath
    releases_author = group_releases_by_author(releases)
    revisions_author = group_revisions_by_author(currentlog)
    params["searchList"] = [{"releases_by_author" : releases_author,
                             "releases" : releases,
                             "revisions_by_author": revisions_author}]
    t = Template(**params)
    return t.respond()


class InvalidEntryError(Exception):
    pass

def parse_repsys_entry(revlog):
    # parse entries in the format:
    # %repsys <operation>
    # key: value
    # ..
    # <newline>
    # <comments>
    #
    if len(revlog.lines) == 0 or not revlog.lines[0].startswith("%repsys"):
        raise InvalidEntryError
    try:       
        data = {"operation" : revlog.lines[0].split()[1]}
    except IndexError:
        raise InvalidEntryError
    for line in revlog.lines[1:]:
        if not line:
            break
        try:
            key, value = line.split(":", 1)
        except ValueError:
            raise InvalidEntryError
        data[key.strip().lower()] = value.strip() # ???
    return data
        

def get_revision_offset():
    try:
        revoffset = config.getint("log", "revision-offset", 0)
    except (ValueError, TypeError):
        raise Error, ("Invalid revision-offset number in configuration "
                      "file(s).")
    return revoffset or 0

oldmsgpat = re.compile(
        r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.")

def parse_markrelease_log(relentry):
    if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \
            or parse_repsys_entry(relentry))):
        raise InvalidEntryError
    from_rev = None
    path = None
    for changed in relentry.changed:
        if changed["action"] == "A" and changed["from_rev"]:
            from_rev = changed["from_rev"]
            path = changed["path"]
            break
    else:
        raise InvalidEntryError
    # get the version and release from the names in the path, do not relay
    # on log messages
    version, release = path.rsplit(os.path.sep, 3)[-2:]
    return version, release, from_rev


def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
        template=None, macros=[], exported=None):
    concat = config.get("log", "concat", "").split()
    revoffset = get_revision_offset()
    svn = SVN()
    pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True)
    pkgcurrenturl = layout.checkout_url(pkgdirurl)
    releaseslog = svn.log(pkgreleasesurl, noerror=1)
    currentlog = svn.log(pkgcurrenturl, limit=size, start=rev,
            end=revoffset)

    # sort releases by copyfrom-revision, so that markreleases for same
    # revisions won't look empty
    releasesdata = []
    if releaseslog:
        for relentry in releaseslog[::-1]:
            try:
                (version, release, relrevision) = \
                        parse_markrelease_log(relentry)
            except InvalidEntryError:
                continue
            releasesdata.append((relrevision, -relentry.revision, relentry, 
                version, release))
        releasesdata.sort()

    # collect valid releases using the versions provided by the changes and
    # the packages
    prevrevision = 0
    releases = []
    for (relrevision, dummy, relentry, version, release) in releasesdata:
        if prevrevision == relrevision: 
            # ignore older markrelease of the same revision, since they
            # will have no history
            continue
        entries = [entry for entry in currentlog
                    if relrevision >= entry.revision and
                      (prevrevision < entry.revision)]
        if not entries:
            #XXX probably a forced release, without commits in current/,
            # check if this is the right behavior
            sys.stderr.write("warning: skipping (possible) release "
                    "%s-%s@%s, no commits since previous markrelease (r%r)\n" %
                    (version, release, relrevision, prevrevision))
            continue

        release = make_release(author=relentry.author,
                revision=relentry.revision, date=relentry.date,
                lines=relentry.lines, entries=entries,
                version=version, release=release)
        releases.append(release)
        prevrevision = relrevision
            
    # look for commits that have been not submitted (released) yet
    # this is done by getting all log entries newer (greater revision no.)
    # than releasesdata[-1] (in the case it exists)
    if releasesdata:
        latest_revision = releasesdata[-1][0] # the latest copied rev
    else:
        latest_revision = 0
    notsubmitted = [entry for entry in currentlog 
                    if entry.revision > latest_revision]
    if notsubmitted:
        # if they are not submitted yet, what we have to do is to add
        # a release/version number from getrelease()
        version, release = getrelease(pkgdirurl, macros=macros,
                exported=exported)
        toprelease = make_release(entries=notsubmitted, released=False,
                        version=version, release=release)
        releases.append(toprelease)

    data = dump_file(releases[::-1], currentlog=currentlog, template=template)
    return data

def _split_changelog(stream):
    current = None
    count = 0
    def finish(entry):
        lines = entry[2]
        # strip newlines at the end
        for i in xrange(len(lines)-1, -1, -1):
            if lines[i] != "\n":
                break
            del lines[i]
        return entry
    for line in stream:
        if line.startswith("*"):
            if current:
                yield finish(current)
            fields = line.split()
            rawdate = " ".join(fields[:5])
            try:
                date = time.strptime(rawdate, "* %a %b %d %Y")
            except ValueError, e:
                raise Error, "failed to parse spec changelog: %s" % e
            curlines = [line]
            current = (date, count, curlines)
            # count used to ensure stable sorting when changelog entries
            # have the same date, otherwise it would also compare the
            # changelog lines
            count -= 1
        elif current:
            curlines.append(line)
        else:
            pass # not good, but ignore
    if current:
        yield finish(current)

def sort_changelog(stream):
    entries = _split_changelog(stream)
    log = StringIO()
    for time, count, elines in sorted(entries, reverse=True):
        log.writelines(elines)
        log.write("\n")
    return log

def split_spec_changelog(stream):
    chlog = StringIO()
    spec = StringIO()
    found = 0
    visible = 0
    for line in stream:
        if line.startswith("%changelog"):
            found = 1
        elif not found:
            spec.write(line)
        elif found:
            if line.strip():
                visible = 1
            chlog.write(line)
        elif line.startswith("%"):
            found = 0
            spec.write(line)
    spec.seek(0)
    if not visible:
        # when there are only blanks in the changelog, make it empty
        chlog = StringIO()
    return spec, chlog

def get_old_log(pkgdirurl):
    chlog = StringIO()
    oldurl = config.get("log", "oldurl")
    if oldurl:
        svn = SVN()
        tmpdir = tempfile.mktemp()
        try:
            if oldurl == '.' or oldurl.startswith('./'):
                pkgoldurl = os.path.join(pkgdirurl, oldurl)
            else:
                pkgname = layout.package_name(pkgdirurl)
                pkgoldurl = os.path.join(oldurl, pkgname)
            try:
                # we're using HEAD here because fixes in misc/ (oldurl) may
                # be newer than packages' last changed revision.
                svn.export(pkgoldurl, tmpdir)
            except Error:
                pass
            else:
                logfile = os.path.join(tmpdir, "log")
                if os.path.isfile(logfile):
                    file = open(logfile)
                    chlog.write("\n") # TODO needed?
                    log = file.read()
                    log = escape_macros(log)
                    chlog.write(log)
                    file.close()
        finally:
            if os.path.isdir(tmpdir):
                shutil.rmtree(tmpdir)
    chlog.seek(0)
    return chlog

def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
        submit=False, sort=False, template=None, macros=[], exported=None,
        oldlog=False):
    """Generates the changelog for a given package URL

    @another:   a stream with the contents of a changelog to be merged with
                the one generated
    @svn:       enable changelog from svn
    @rev:       generate the changelog with the changes up to the given
                revision
    @size:      the number of revisions to be used (as in svn log --limit)
    @submit:    defines whether the latest unreleased log entries should have
                the version parsed from the spec file
    @sort:      should changelog entries be reparsed and sorted after appending
                the oldlog?
    @template:  the path to the cheetah template used to generate the
                changelog from svn
    @macros:    a list of tuples containing macros to be defined when
                parsing the version in the changelog
    @exported:  the path of a directory containing an already existing
                checkout of the package, so that the spec file can be
                parsed from there
    @oldlog:    if set it will try to append the old changelog file defined
                in oldurl in mgarepo.conf
    """
    newlog = StringIO()
    if svn:
        rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
                template=template, macros=macros, exported=exported)
        newlog.write(rawsvnlog)
    if another:
        newlog.writelines(another)
    if oldlog:
        newlog.writelines(get_old_log(pkgdirurl))
    if sort:
        newlog.seek(0)
        newlog = sort_changelog(newlog)
    newlog.seek(0)
    return newlog

def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
        submit=False, sort=False, template=None, macros=[], exported=None):
    fi = open(specfile)
    spec, oldchlog = split_spec_changelog(fi)
    fi.close()
    another = None
    if config.getbool("log", "merge-spec", False):
        another = oldchlog
    sort = sort or config.getbool("log", "sort", False)
    chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size,
                submit=submit, sort=sort, template=template, macros=macros,
                exported=exported, oldlog=True)
    fo = open(specfile, "w")
    fo.writelines(spec)
    fo.write("\n\n%changelog\n")
    fo.writelines(chlog)
    fo.close()

if __name__ == "__main__":
    l = svn2rpm(sys.argv[1])
    print l

# vim:et:ts=4:sw=4