make the V1_6_X branch the trunk

1 files changed, 194 insertions, 92 deletions

diff --git a/RepSys/log.py b/RepSys/log.py
index 2c69693..a1d1944 100644
--- a/RepSys/log.py
+++ b/RepSys/log.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-from RepSys import Error, config, RepSysTree
+from RepSys import Error, config, layout
 from RepSys.svn import SVN
 from RepSys.util import execcmd
 
@@ -8,30 +8,50 @@ try:
 except ImportError:
     raise Error, "repsys requires the package python-cheetah"
 
+from cStringIO import StringIO
+
 import sys
 import os
 import re
 import time
 import locale
-import codecs
 import glob
 import tempfile
 import shutil
 
+
+locale.setlocale(locale.LC_ALL, "C")
+
 default_template = """
+#if not $releases_by_author[-1].visible
+  ## Hide the first release that contains no changes. It must be a
+  ## reimported package and the log gathered from misc/ already should
+  ## contain a correct entry for the version-release:
+  #set $releases_by_author = $releases_by_author[:-1]
+#end if
 #for $rel in $releases_by_author
 * $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
- ##
- #if not $rel.released
-  (not released yet)
++ Revision: $rel.revision
+## #if not $rel.released
+##+ Status: not released
+## #end if
+ #if not $rel.visible
++ rebuild (emptylog)
  #end if
  #for $rev in $rel.release_revisions
   #for $line in $rev.lines
-  $line
+$line
   #end for
  #end for
 
  #for $author in $rel.authors
+  #if not $author.visible
+    #continue
+  #end if
+  ##alternatively, one could use:
+  ###if $author.email == "root"
+  ## #continue
+  ###end if
   + $author.name <$author.email>
   #for $rev in $author.revisions
     #for $line in $rev.lines
@@ -50,14 +70,13 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None):
     Is here where things should be changed if "automatic release increasing" 
     will be used.
     """
-    svn = SVN()
     from RepSys.rpmutil import rpm_macros_defs
-    tmpdir = tempfile.mktemp()
+    svn = SVN()
     pkgcurrenturl = os.path.join(pkgdirurl, "current")
     specurl = os.path.join(pkgcurrenturl, "SPECS")
     if exported is None:
         tmpdir = tempfile.mktemp()
-        svn.export(specurl, tmpdir, revision=SVN.makerev(rev))
+        svn.export(specurl, tmpdir, rev=rev)
     else:
         tmpdir = os.path.join(exported, "SPECS")
     try:
@@ -89,8 +108,7 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None):
         if exported is None and os.path.isdir(tmpdir):
             shutil.rmtree(tmpdir)
             
-
-class ChangelogRevision:
+class _Revision:
     lines = []
     date = None
     raw_date = None
@@ -103,11 +121,12 @@ class ChangelogRevision:
 
     def __repr__(self):
         lines = repr(self.lines)[:30] + "...]" 
-        line = "<ChangelogRevision %d author=%r date=%r lines=%s>" % \
+        line = "<_Revision %d author=%r date=%r lines=%s>" % \
                     (self.revision, self.author, self.date, lines)
         return line
 
-class ChangelogRelease(ChangelogRevision):
+
+class _Release(_Revision):
     version = None
     release = None
     revisions = []
@@ -116,11 +135,11 @@ class ChangelogRelease(ChangelogRevision):
     visible = False
 
     def __init__(self, **kwargs):
-        ChangelogRevision.__init__(self, **kwargs)
         self.revisions = []
+        _Revision.__init__(self, **kwargs)
 
     def __repr__(self):
-        line = "<ChangelogRelease v=%s r=%s revs=%r>" % \
+        line = "<_Release v=%s r=%s revs=%r>" % \
                     (self.version, self.release, self.revisions)
         return line
 
@@ -153,59 +172,62 @@ def format_lines(lines):
     return entrylines
 
 
-class ChangelogByAuthor:
+class _Author:
     name = None
     email = None
     revisions = None
+    visible = False
 
 
 def group_releases_by_author(releases):
     allauthors = []
     grouped = []
     for release in releases:
+
+        # group revisions of the release by author
         authors = {}
         latest = None
         for revision in release.revisions:
             authors.setdefault(revision.author, []).append(revision)
 
-        # all the mess below is to sort by author and by revision number
+        # create _Authors and sort them by their latest revisions
         decorated = []
         for authorname, revs in authors.iteritems():
-            author = ChangelogByAuthor()
+            author = _Author()
             author.name = revs[0].author_name
             author.email = revs[0].author_email
-            revdeco = [(r.revision, r) for r in revs]
-            revdeco.sort(reverse=1)
-            author.revisions = [t[1] for t in revdeco]
+            author.revisions = revs
+            # #41117: mark those authors without visible messages
+            author.visible = bool(sum(len(rev.lines) for rev in revs))
             revlatest = author.revisions[0]
-            # keep the latest revision even for silented authors (below)
+            # keep the latest revision even for completely invisible
+            # authors (below)
             if latest is None or revlatest.revision > latest.revision:
                 latest = revlatest
-            count = sum(len(rev.lines) for rev in author.revisions)
-            if count == 0:
-                # skipping author with only silented lines
+            if not author.visible:
+                # only sort those visible authors, invisible ones are used
+                # only in "latest"
                 continue
-            decorated.append((revdeco[0][0], author))
+            decorated.append((revlatest.revision, author))
+        decorated.sort(reverse=1)
 
-        if not decorated:
-            # skipping release with only authors with silented lines
-            continue
+        if release.visible:
+            release.authors = [t[1] for t in decorated]
+            firstrel, release.authors = release.authors[0], release.authors[1:]
+            release.author_name = firstrel.name
+            release.author_email = firstrel.email
+            release.release_revisions = firstrel.revisions
+        else:
+            # we don't care about other possible authors in completely
+            # invisible releases
+            firstrev = release.revisions[0]
+            release.author_name = firstrev.author_name
+            release.author_email = firstrev.author_email
+            release.raw_date = firstrev.raw_date
+            release.date = firstrev.date
 
-        decorated.sort(reverse=1)
-        release.authors = [t[1] for t in decorated]
-        # the difference between a released and a not released _Release is
-        # the way the release numbers is obtained. So, when this is a
-        # released, we already have it, but if we don't, we should get de
-        # version/release string using getrelease and then get the first
-        first, release.authors = release.authors[0], release.authors[1:]
-        release.author_name = first.name
-        release.author_email = first.email
-        release.release_revisions = first.revisions
-
-        #release.date = first.revisions[0].date
         release.date = latest.date
         release.raw_date = latest.raw_date
-        #release.revision = first.revisions[0].revision
         release.revision = latest.revision
 
         grouped.append(release)
@@ -217,7 +239,7 @@ def group_revisions_by_author(currentlog):
     revisions = []
     last_author = None
     for entry in currentlog:
-        revision = ChangelogRevision()
+        revision = _Revision()
         revision.lines = format_lines(entry.lines)
         revision.raw_date = entry.date
         revision.date = parse_raw_date(entry.date)
@@ -225,7 +247,7 @@ def group_revisions_by_author(currentlog):
         if entry.author == last_author:
             revisions[-1].revisions.append(revision)
         else:
-            author = ChangelogByAuthor()
+            author = _Author()
             author.name, author.email = get_author_name(entry.author)
             author.revisions = [revision]
             revisions.append(author)
@@ -269,7 +291,7 @@ def filter_log_lines(lines):
 
 def make_release(author=None, revision=None, date=None, lines=None,
         entries=[], released=True, version=None, release=None):
-    rel = ChangelogRelease()
+    rel = _Release()
     rel.author = author
     if author:
         rel.author_name, rel.author_email = get_author_name(author)
@@ -282,11 +304,11 @@ def make_release(author=None, revision=None, date=None, lines=None,
     rel.visible = False
     for entry in entries:
         lines = filter_log_lines(entry.lines)
-        if lines:
-            rel.visible = True
-        revision = ChangelogRevision()
+        revision = _Revision()
         revision.revision = entry.revision
         revision.lines = format_lines(lines)
+        if revision.lines:
+            rel.visible = True
         revision.date = parse_raw_date(entry.date)
         revision.raw_date = entry.date
         revision.author = entry.author
@@ -297,7 +319,8 @@ def make_release(author=None, revision=None, date=None, lines=None,
 
 
 def dump_file(releases, currentlog=None, template=None):
-    templpath = template or config.get("template", "path", None)
+    templpath = template or config.get("template", "path",
+            "/usr/share/repsys/default.chlog")
     params = {}
     if templpath is None or not os.path.exists(templpath):
         params["source"] = default_template
@@ -311,12 +334,7 @@ def dump_file(releases, currentlog=None, template=None):
                              "releases" : releases,
                              "revisions_by_author": revisions_author}]
     t = Template(**params)
-    chlog = t.respond()
-    try:
-        chlog = chlog.decode("utf8")
-    except UnicodeError:
-        pass
-    return chlog
+    return t.respond()
 
 
 class InvalidEntryError(Exception):
@@ -379,18 +397,14 @@ def parse_markrelease_log(relentry):
 
 def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
         template=None, macros=[], exported=None):
-    size = size or 0
     concat = config.get("log", "concat", "").split()
     revoffset = get_revision_offset()
     svn = SVN()
-    pkgreleasesurl = os.path.join(pkgdirurl, "releases")
-    pkgcurrenturl = os.path.join(pkgdirurl, "current")
-    releaseslog = list(svn.log(pkgreleasesurl, 
-            strict_node_history=False, noerror=1)) or []
-    currentlog = list(svn.log(pkgcurrenturl, 
-            strict_node_history=False,
-            revision_start=SVN.makerev(rev),
-            revision_end=SVN.makerev(revoffset), limit=size))
+    pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True)
+    pkgcurrenturl = layout.checkout_url(pkgdirurl)
+    releaseslog = svn.log(pkgreleasesurl, noerror=1)
+    currentlog = svn.log(pkgcurrenturl, limit=size, start=rev,
+            end=revoffset)
 
     # sort releases by copyfrom-revision, so that markreleases for same
     # revisions won't look empty
@@ -454,40 +468,74 @@ def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
     data = dump_file(releases[::-1], currentlog=currentlog, template=template)
     return data
 
-
-
-def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
-        submit=False, template=None, macros=[], exported=None):
-    newlines = []
+def _split_changelog(stream):
+    current = None
+    count = 0
+    def finish(entry):
+        lines = entry[2]
+        # strip newlines at the end
+        for i in xrange(len(lines)-1, -1, -1):
+            if lines[i] != "\n":
+                break
+            del lines[i]
+        return entry
+    for line in stream:
+        if line.startswith("*"):
+            if current:
+                yield finish(current)
+            fields = line.split()
+            rawdate = " ".join(fields[:5])
+            try:
+                date = time.strptime(rawdate, "* %a %b %d %Y")
+            except ValueError, e:
+                raise Error, "failed to parse spec changelog: %s" % e
+            curlines = [line]
+            current = (date, count, curlines)
+            # count used to ensure stable sorting when changelog entries
+            # have the same date, otherwise it would also compare the
+            # changelog lines
+            count -= 1
+        elif current:
+            curlines.append(line)
+        else:
+            pass # not good, but ignore
+    if current:
+        yield finish(current)
+
+def sort_changelog(stream):
+    entries = _split_changelog(stream)
+    log = StringIO()
+    for time, count, elines in sorted(entries, reverse=True):
+        log.writelines(elines)
+        log.write("\n")
+    return log
+
+def split_spec_changelog(stream):
+    chlog = StringIO()
+    spec = StringIO()
     found = 0
-
-    encoding = locale.getpreferredencoding()
-
-    def open(name, mode="r"):
-        return codecs.open(name, mode, encoding, errors="replace")
-    
-    # Strip old changelogs
-    for line in open(specfile):
+    for line in stream:
         if line.startswith("%changelog"):
             found = 1
         elif not found:
-            newlines.append(line)
+            spec.write(line)
+        elif found:
+            chlog.write(line)
         elif line.startswith("%"):
             found = 0
-            newlines.append(line)
-
-    # Create new changelog
-    newlines.append("\n\n%changelog\n")
-    newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
-        template=template, macros=macros, exported=exported))
+            spec.write(line)
+    spec.seek(0)
+    chlog.seek(0)
+    return spec, chlog
 
-    # Merge old changelog, if available
+def get_old_log(pkgdirurl):
+    chlog = StringIO()
     oldurl = config.get("log", "oldurl")
     if oldurl:
         svn = SVN()
         tmpdir = tempfile.mktemp()
         try:
-            pkgname = RepSysTree.pkgname(pkgdirurl)
+            pkgname = layout.package_name(pkgdirurl)
             pkgoldurl = os.path.join(oldurl, pkgname)
             try:
                 # we're using HEAD here because fixes in misc/ (oldurl) may
@@ -499,20 +547,74 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
                 logfile = os.path.join(tmpdir, "log")
                 if os.path.isfile(logfile):
                     file = open(logfile)
-                    newlines.append("\n")
+                    chlog.write("\n") # TODO needed?
                     log = file.read()
                     log = escape_macros(log)
-                    newlines.append(log)
+                    chlog.write(log)
                     file.close()
         finally:
             if os.path.isdir(tmpdir):
                 shutil.rmtree(tmpdir)
+    chlog.seek(0)
+    return chlog
 
-    # Write new specfile
-    file = open(specfile, "w")
-    file.write("".join(newlines))
-    file.close()
+def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None,
+        submit=False, sort=False, template=None, macros=[], exported=None,
+        oldlog=False):
+    """Generates the changelog for a given package URL
+
+    @another:   a stream with the contents of a changelog to be merged with
+                the one generated
+    @svn:       enable changelog from svn
+    @rev:       generate the changelog with the changes up to the given
+                revision
+    @size:      the number of revisions to be used (as in svn log --limit)
+    @submit:    defines whether the latest unreleased log entries should have
+                the version parsed from the spec file
+    @sort:      should changelog entries be reparsed and sorted after appending
+                the oldlog?
+    @template:  the path to the cheetah template used to generate the
+                changelog from svn
+    @macros:    a list of tuples containing macros to be defined when
+                parsing the version in the changelog
+    @exported:  the path of a directory containing an already existing
+                checkout of the package, so that the spec file can be
+                parsed from there
+    @oldlog:    if set it will try to append the old changelog file defined
+                in oldurl in repsys.conf
+    """
+    newlog = StringIO()
+    if svn:
+        rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
+                template=template, macros=macros, exported=exported)
+        newlog.write(rawsvnlog)
+    if another:
+        newlog.writelines(another)
+    if oldlog:
+        newlog.writelines(get_old_log(pkgdirurl))
+    if sort:
+        newlog.seek(0)
+        newlog = sort_changelog(newlog)
+    newlog.seek(0)
+    return newlog
 
+def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
+        submit=False, sort=False, template=None, macros=[], exported=None):
+    fi = open(specfile)
+    spec, oldchlog = split_spec_changelog(fi)
+    fi.close()
+    another = None
+    if config.getbool("log", "merge-spec", False):
+        another = oldchlog
+    sort = sort or config.getbool("log", "sort", False)
+    chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size,
+                submit=submit, sort=sort, template=template, macros=macros,
+                exported=exported, oldlog=True)
+    fo = open(specfile, "w")
+    fo.writelines(spec)
+    fo.write("\n\n%changelog\n")
+    fo.writelines(chlog)
+    fo.close()
 
 if __name__ == "__main__":
     l = svn2rpm(sys.argv[1])