Frontported changes from V1_6_X since april

26 files changed, 996 insertions, 202 deletions

diff --git a/CHANGES b/CHANGES
index aa9e004..ce1fa7e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,36 @@
+* 1.6.18
+- initialize plugins in create-srpm too
+
+* 1.6.17
+- brought from mdvsys world the sync command
+- ldapusers: the configuration format has changed, now it uses python
+  template strings
+- ldapusers: many fixes: better error messages, ldap-port working, results
+  contain only the fields needed, unbinding after search, filters are
+  escaped
+
+* 1.6.16
+- introduced the plugin ldapusers: repsys user data obtained from LDAP;
+  this plugin is builtin
+- added support to plugins, and the hability to wrap configuration sections
+- added workaround in the template to ignore empty releases
+- added initial support to mirrors, as requested by mrl; it required the
+  new subcommand "ci"
+- changelogs from misc/ will now should come from HEAD and should be
+  escaped (%%)
+
+* 1.6.15
+- empty changelog entries are now shown, with a EMPTYLOG tag to allow
+  rpmlint warn the developer about it
+- check (and warn) if a temporary package has already been removed before
+  trying to remove it
+
+* 1.6.2b
+- make submit pass --define options to create-srpm script
+- print error message when create-srpm fails
+- make get_srpm return the srpms list
+- add upload-srpm support in create-srpm
+
 * 1.6.2a
 - moved revision-offset to [log] section and added a comment
 
diff --git a/MANIFEST.in b/MANIFEST.in
index 9311d09..1264d01 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,6 @@
 recursive-include RepSys *.py
-include repsys repsys.conf MANIFEST.in
+include RepSys/plugins/*.txt
+include repsys repsys.conf MANIFEST.in 
+include README.LDAP
+include *.chlog
+include rebrand-mdk create-srpm getsrpm-mdk
diff --git a/Makefile b/Makefile
index 46ddead..e1aa91b 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,7 @@ RELEASE:=$(shell rpm -q --qf %{RELEASE} --specfile $(PACKAGE).spec)
 TAG := $(shell echo "V$(VERSION)_$(RELEASE)" | tr -- '-.' '__')
 
 FILES = ChangeLog Makefile MANIFEST.in PKG-INFO create-srpm getsrpm-mdk rebrand-mdk \
+        {compatv15,default,oldfashion,revno}.chlog \
         repsys repsys.conf repsys.spec setup.cfg setup.py RepSys/*.py RepSys/{cgi,commands}/*.py
 
 # rules to build a test rpm
diff --git a/README.LDAP b/README.LDAP
new file mode 100644
index 0000000..863be6d
--- /dev/null
+++ b/README.LDAP
@@ -0,0 +1,52 @@
+A Repsys plugin for obtaining users from a LDAP server.
+
+In order to enable the plugin, the user must define the following 
+options in the [global] section of repsys.conf:
+
+    ldap-server [required]
+        the host name of the LDAP server
+    ldap-port [optional] [default: 389]
+        the port of the LDAP server
+    ldap-base [required]
+        the base DN where the search will be performed
+    ldap-binddn [optional] [default: empty]
+        the DN used to bind
+    ldap-bindpw [optional] [default: empty]
+        the password used to bind
+    ldap-filterformat [optional] 
+            [default: (&(objectClass=inetOrgPerson)(uid=$username))]
+        RFC-2254 filter string used in the search of the user entry.
+        Note that this is a python template string and will have the 
+        user name as parameter. For example:
+
+           ldap-filterformat = (&(objectClass=inetOrgPerson)(uid=$username))
+
+        Will result in the search filter:
+
+           (&(objectClass=inetOrgPerson)(uid=john))
+
+    ldap-resultformat [optional] [default: $cn <$mail>]
+        This is a python template string. This string will be 
+        formatted using one dict object containing the fields
+        returned in the LDAP search, for example:
+
+          >>> format = Template("$cn <$mail>")
+          >>> d = search(basedn, filter)
+          >>> d
+          {"cn": "John Doe", "mail": "john@mandriva.org", 
+           "uidNumber": "1290", "loginShell": "/bin/bash", 
+            ... many other attributes ... }
+          >>> value = format.substitute(d)
+          >>> print value
+          John Doe <john@mandriva.org>
+
+        Note that only the first value of the attributes will be 
+        used.
+
+When the searched option is not found, it will try in repsys.conf. All
+the values found.  (including from repsys.conf) will be cached between
+each configuration access.
+
+This plugin requires the package python-ldap.
+
+For more information, look http://qa.mandriva.com/show_bug.cgi?id=30549
diff --git a/RepSys/ConfigParser.py b/RepSys/ConfigParser.py
index 0f219b9..4dc3e3c 100644
--- a/RepSys/ConfigParser.py
+++ b/RepSys/ConfigParser.py
@@ -348,6 +348,7 @@ import os
 class Config:
     def __init__(self):
         self._config = ConfigParser()
+        self._wrapped = {}
         conffiles = []
         conffiles.append("/etc/repsys.conf")
         repsys_conf = os.environ.get("REPSYS_CONF")
@@ -358,6 +359,14 @@ class Config:
             if os.path.isfile(file):
                 self._config.read(file)
 
+    def wrap(self, section, handler, option=None):
+        """Set one wrapper for a given section
+        
+        The wrapper must be a function 
+        f(section, option=None, default=None, walk=False).
+        """
+        self._wrapped[section] = handler
+
     def sections(self):
         try:
             return self._config.sections()
@@ -373,12 +382,20 @@ class Config:
     def set(self, section, option, value):
         return self._config.set(section, option, value)
     
-    def walk(self, section):
-        return self._config.walk(section)
-
-    def get(self, section, option, default=None):
+    def walk(self, section, option=None, raw=0, vars=None):
+        handler = self._wrapped.get(section)
+        if handler:
+            return handler(section, option, walk=True)
+        return self._config.walk(section, option, raw, vars)
+
+    def get(self, section, option, default=None, raw=False, wrap=True):
+        if wrap:
+            handler = self._wrapped.get(section)
+            if handler:
+                handler = self._wrapped.get(section)
+                return handler(section, option, default)
         try:
-            return self._config.get(section, option)
+            return self._config.get(section, option, raw=raw)
         except Error:
             return default
     
@@ -395,4 +412,22 @@ class Config:
             return states[ret.lower()]
         return default
 
+def test():
+    config = Config()
+    def handler(section, option=None, default=None, walk=False):
+        d = {"fulano": "ciclano",
+             "foolano": "ceeclano"}
+        if walk:
+            return d.items()
+        elif option in d:
+            return d[option]
+        else:
+            return config.get(section, option, default, wrap=False)
+    config.wrap("users", handler=handler)
+    print config.get("users", "fulano") # found in wrapper
+    print config.get("users", "andreas") # found in repsys.conf
+    print config.walk("users")
+
+if __name__ == "__main__":
+    test()
 # vim:ts=4:sw=4:et
diff --git a/RepSys/__init__.py b/RepSys/__init__.py
index c371e7a..b303065 100644
--- a/RepSys/__init__.py
+++ b/RepSys/__init__.py
@@ -1,9 +1,30 @@
 #!/usr/bin/python
+import re
+import os
+import tempfile
 
 import ConfigParser
+
 config = ConfigParser.Config()
+tempfile.tempdir = config.get("global", "tempdir", None) or None # when ""
 del ConfigParser
 
 class Error(Exception): pass
 
+class RepSysTree:
+    """
+    This class just hold methods that abstract all the not-so-explicit
+    rules about the directory structure of a repsys repository.
+    """
+    def fixpath(cls, url):
+        return re.sub("/+$", "", url)
+    fixpath = classmethod(fixpath)
+
+    def pkgname(cls, pkgdirurl):
+        # we must remove trailling slashes in the package path because
+        # os.path.basename could return "" from URLs ending with "/"
+        fixedurl = cls.fixpath(pkgdirurl)
+        return os.path.basename(fixedurl)
+    pkgname = classmethod(pkgname)
+            
 # vim:et:ts=4:sw=4
diff --git a/RepSys/cgiutil.py b/RepSys/cgiutil.py
index 6dda91e..35c5efb 100644
--- a/RepSys/cgiutil.py
+++ b/RepSys/cgiutil.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 from RepSys import Error, config
 from RepSys.svn import SVN
+from RepSys.ConfigParser import NoSectionError
 import time
 import re
 
@@ -10,11 +11,23 @@ class SubmitTarget:
     def __init__(self):
         self.name = ""
         self.target = ""
+        self.macros = []
         self.allowed = []
         self.scripts = []
 
 TARGETS = []
 
+def parse_macrosref(refs, config):
+    macros = []
+    for name in refs:
+        secname = "macros %s" % name
+        try:
+            macros.extend(config.walk(secname, raw=True))
+        except NoSectionError:
+            raise Error, "missing macros section " \
+                    "%r in configuration" % secname
+    return macros
+
 def get_targets():
     global TARGETS
     if not TARGETS:
@@ -27,12 +40,11 @@ def get_targets():
                 target = SubmitTarget()
                 target.name = m.group(1)
                 for option, value in config.walk(section):
-                    if option == "target":
-                        target.target = value.split()
-                    elif option == "allowed":
-                        target.allowed = value.split()
-                    elif option == "scripts":
-                        target.scripts = value.split()
+                    if option in ("target", "allowed", "scripts"):
+                        setattr(target, option, value.split())
+                    elif option == "rpm-macros":
+                        refs = value.split()
+                        target.macros = parse_macrosref(refs, config)
                     else:
                         raise Error, "unknown [%s] option %s" % (section, option)
                 TARGETS.append(target)
diff --git a/RepSys/commands/changed.py b/RepSys/commands/changed.py
index c99f3ae..d3094a8 100644
--- a/RepSys/commands/changed.py
+++ b/RepSys/commands/changed.py
@@ -25,7 +25,7 @@ def parse_options():
     opts, args = parser.parse_args()
     if len(args) != 1:
         raise Error, "invalid arguments"
-    opts.url = default_parent(args[0])
+    opts.pkgdirurl = default_parent(args[0])
     opts.verbose = 1 # Unconfigurable
     return opts
 
diff --git a/RepSys/commands/ci.py b/RepSys/commands/ci.py
new file mode 100644
index 0000000..9ffa3bd
--- /dev/null
+++ b/RepSys/commands/ci.py
@@ -0,0 +1,29 @@
+#!/usr/bin/python
+from RepSys.command import *
+from RepSys.rpmutil import commit
+
+HELP = """\
+Usage: repsys ci [TARGET]
+
+Will commit a change. The difference between an ordinary "svn ci" and
+"repsys ci" is that it relocates the working copy to the default repository
+in case the option "mirror" is set in repsys.conf.
+
+Options:
+    -h      Show this message
+
+Examples:
+    repsys ci
+    repsys ci SPECS/package.spec SPECS/package-patch.patch
+"""
+
+def parse_options():
+    parser = OptionParser(help=HELP)
+    parser.add_option("-m", dest="message", default=None)
+    opts, args = parser.parse_args()
+    if len(args):
+        opts.target = args[0]
+    return opts
+
+def main():
+    do_command(parse_options, commit)
diff --git a/RepSys/commands/co.py b/RepSys/commands/co.py
index 0c9d2dc..f2b4d64 100644
--- a/RepSys/commands/co.py
+++ b/RepSys/commands/co.py
@@ -23,7 +23,7 @@ def parse_options():
     opts, args = parser.parse_args()
     if len(args) not in (1, 2):
         raise Error, "invalid arguments"
-    opts.url = default_parent(args[0])
+    opts.pkgdirurl = default_parent(args[0])
     if len(args) == 2:
         opts.path = args[1]
     else:
diff --git a/RepSys/commands/editlog.py b/RepSys/commands/editlog.py
index a98b761..367238f 100644
--- a/RepSys/commands/editlog.py
+++ b/RepSys/commands/editlog.py
@@ -30,7 +30,8 @@ def parse_options():
 
 def editlog(pkgdirurl, revision):
     svn = SVN()
-    svn.propedit("svn:log", pkgdirurl, revision=revision, revprop=True)
+    svn.propedit("svn:log", pkgdirurl, revision=SVN.makerev(revision),
+            revprop=True)
 
 def main():
     do_command(parse_options, editlog)
diff --git a/RepSys/commands/markrelease.py b/RepSys/commands/markrelease.py
index 9e52a31..440775b 100644
--- a/RepSys/commands/markrelease.py
+++ b/RepSys/commands/markrelease.py
@@ -9,7 +9,7 @@
 #
 from RepSys import Error
 from RepSys.command import *
-from RepSys.rpm import SRPM
+from RepSys.simplerpm import SRPM
 from RepSys.rpmutil import mark_release
 from RepSys.util import get_auth
 import getopt
diff --git a/RepSys/commands/submit.py b/RepSys/commands/submit.py
index 6e9eb0f..5c95526 100644
--- a/RepSys/commands/submit.py
+++ b/RepSys/commands/submit.py
@@ -18,12 +18,18 @@ import xmlrpclib
 HELP = """\
 Usage: repsys submit [OPTIONS] [URL [REVISION]]
 
+Submits the package from URL to the submit host.
+
 Options:
     -t TARGET  Submit given package URL to given target
     -l         Just list available targets
     -r REV     Provides a revision number (when not providing as an
                argument)
+    -s         The host in which the package URL will be submitted
+               (defaults to the host in the URL)
     -h         Show this message
+    --define   Defines one variable to be used by the submit scripts 
+               in the submit host
 
 Examples:
     repsys submit
@@ -39,6 +45,9 @@ def parse_options():
     parser.add_option("-t", dest="target", default="Cooker")
     parser.add_option("-l", dest="list", action="store_true")
     parser.add_option("-r", dest="revision", type="string", nargs=1)
+    parser.add_option("-s", dest="submithost", type="string", nargs=1,
+            default=None)
+    parser.add_option("--define", action="append")
     opts, args = parser.parse_args()
     if not args:
         name, rev = get_submit_info(".")
@@ -63,61 +72,33 @@ def parse_options():
         raise Error, "provide -l or a revision number"
     return opts
 
-def submit(pkgdirurl, revision, target, list=0):
+def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None):
     #if not NINZ:
     #    raise Error, "you must have NINZ installed to use this command"
-    type, rest = urllib.splittype(pkgdirurl)
-    host, path = urllib.splithost(rest)
-    user, host = urllib.splituser(host)
-    host, port = urllib.splitport(host)
-    if type != "https" and type != "svn+ssh":
-        raise Error, "you must use https:// or svn+ssh:// urls"
-    if user:
-        user, passwd = urllib.splitpasswd(user)
-        if passwd:
-            raise Error, "do not use a password in your command line"
-    if type == "https":
-        user, passwd = get_auth(username=user)
-        #soap = NINZ.client.Binding(host=host,
-        #                           url="https://%s/scripts/cnc/soap" % host,
-        #                           ssl=1,
-        #                           auth=(NINZ.client.AUTH.httpbasic,
-        #                                 user, passwd))
-        if port:
-            port = ":"+port
-        else:
-            port = ""
-        iface = xmlrpclib.ServerProxy("https://%s:%s@%s%s/scripts/cnc/xmlrpc"
-                                      % (user, passwd, host, port))
-        try:
-            if list:
-                targets = iface.submit_targets()
-                if not targets:
-                    raise Error, "no targets available"
-                sys.stdout.writelines(['"%s"\n' % x for x in targets])
-            else:
-                iface.submit_package(pkgdirurl, revision, target)
-                print "Package submitted!"
-        #except NINZ.client.SoapError, e:
-        except xmlrpclib.ProtocolError, e:
-            raise Error, "remote error: "+str(e.errmsg)
-        except xmlrpclib.Fault, e:
-            raise Error, "remote error: "+str(e.faultString)
-        except xmlrpclib.Error, e:
-            raise Error, "remote error: "+str(e)
+    if submithost is None:
+        submithost = config.get("submit", "host")
+        if submithost is None:
+            # extract the submit host from the svn host
+            type, rest = urllib.splittype(pkgdirurl)
+            host, path = urllib.splithost(rest)
+            user, host = urllib.splituser(host)
+            submithost, port = urllib.splitport(host)
+            del type, user, port, path, rest
+    # runs a create-srpm in the server through ssh, which will make a
+    # copy of the rpm in the export directory
+    if list:
+        raise Error, "unable to list targets from svn+ssh:// URLs"
+    createsrpm = get_helper("create-srpm")
+    command = "ssh %s %s '%s' -r %s -t %s" % (
+            submithost, createsrpm, pkgdirurl, revision, target)
+    if define:
+        command += " " + " ".join([ "--define " + x for x in define ])
+    status, output = execcmd(command)
+    if status == 0:
+        print "Package submitted!"
     else:
-        # runs a create-srpm in the server through ssh, which will make a
-        # copy of the rpm in the export directory
-        if list:
-            raise Error, "unable to list targets from svn+ssh:// URLs"
-        createsrpm = get_helper("create-srpm")
-        command = "ssh %s %s '%s' -r %s -t %s" % (
-                host, createsrpm, pkgdirurl, revision, target)
-        status, output = execcmd(command)
-        if status == 0:
-            print "Package submitted!"
-        else:
-            sys.exit(status)
+        sys.stderr.write(output)
+        sys.exit(status)
 
 
 def main():
diff --git a/RepSys/commands/sync.py b/RepSys/commands/sync.py
new file mode 100644
index 0000000..42ede8d
--- /dev/null
+++ b/RepSys/commands/sync.py
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+from RepSys.command import *
+from RepSys.rpmutil import sync
+
+HELP = """\
+Usage: repsys sync
+
+Will add or removed from the working copy new files added or removed
+from the spec file.
+
+"No changes are commited."
+
+Options:
+    --dry-run    Print results without changing the working copy
+    -h           Show this message
+
+Examples:
+    repsys sync
+"""
+
+def parse_options():
+    parser = OptionParser(help=HELP)
+    parser.add_option("--dry-run", dest="dryrun", default=False,
+            action="store_true")
+    opts, args = parser.parse_args()
+    if len(args):
+        opts.target = args[0]
+    return opts
+
+def main():
+    do_command(parse_options, sync)
diff --git a/RepSys/log.py b/RepSys/log.py
index 5b33c6e..8035997 100644
--- a/RepSys/log.py
+++ b/RepSys/log.py
@@ -1,9 +1,12 @@
 #!/usr/bin/python
-from RepSys import Error, config
+from RepSys import Error, config, RepSysTree
 from RepSys.svn import SVN
 from RepSys.util import execcmd
 
-from Cheetah.Template import Template
+try:
+    from Cheetah.Template import Template
+except ImportError:
+    raise Error, "repsys requires the package python-cheetah"
 
 import sys
 import os
@@ -15,8 +18,6 @@ import tempfile
 import shutil
 
 
-locale.setlocale(locale.LC_ALL, "C")
-
 default_template = """
 #for $rel in $releases_by_author
 * $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
@@ -42,7 +43,7 @@ default_template = """
 #end for
 """
 
-def getrelease(pkgdirurl, rev=None):
+def getrelease(pkgdirurl, rev=None, macros=[]):
     """Tries to obtain the version-release of the package for a 
     yet-not-markrelease revision of the package.
 
@@ -50,28 +51,37 @@ def getrelease(pkgdirurl, rev=None):
     will be used.
     """
     svn = SVN()
+    from RepSys.rpmutil import rpm_macros_defs
     tmpdir = tempfile.mktemp()
     try:
-        pkgname = os.path.basename(pkgdirurl)
+        pkgname = RepSysTree.pkgname(pkgdirurl)
         pkgcurrenturl = os.path.join(pkgdirurl, "current")
         specurl = os.path.join(pkgcurrenturl, "SPECS")
         if svn.exists(specurl):
-            svn.export(specurl, tmpdir, revision=SVN.revision(rev))
+            svn.export(specurl, tmpdir, revision=SVN.makerev(rev))
             found = glob.glob(os.path.join(tmpdir, "*.spec"))
             if found:
                 specpath = found[0]
-                command = (("rpm -q --qf '%%{VERSION}-%%{RELEASE}\n' "
-                           "--specfile %s") % specpath)
+                options = rpm_macros_defs(macros)
+                command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' "
+                           "--specfile %s %s 2>/dev/null") % 
+                           (specpath, options))
                 status, output = execcmd(command)
                 if status != 0:
                     raise Error, "Error in command %s: %s" % (command, output)
                 releases = output.split()
                 try:
-                    version, release = releases[0].split("-", 1)
+                    epoch, vr = releases[0].split(":", 1)
+                    version, release = vr.split("-", 1)
                 except ValueError:
                     raise Error, "Invalid command output: %s: %s" % \
                             (command, output)
-                return version, release
+                #XXX check if this is the right way:
+                if epoch == "(none)":
+                    ev = version
+                else:
+                    ev = epoch + ":" + version
+                return ev, release
     finally:
         if os.path.isdir(tmpdir):
             shutil.rmtree(tmpdir)
@@ -88,16 +98,34 @@ class ChangelogRevision:
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
 
+    def __repr__(self):
+        lines = repr(self.lines)[:30] + "...]" 
+        line = "<ChangelogRevision %d author=%r date=%r lines=%s>" % \
+                    (self.revision, self.author, self.date, lines)
+        return line
 
 class ChangelogRelease(ChangelogRevision):
     version = None
     release = None
-    revisions = None
+    revisions = []
+    release_revisions = []
+    authors = []
+    visible = False
 
     def __init__(self, **kwargs):
         ChangelogRevision.__init__(self, **kwargs)
         self.revisions = []
 
+    def __repr__(self):
+        line = "<ChangelogRelease v=%s r=%s revs=%r>" % \
+                    (self.version, self.release, self.revisions)
+        return line
+
+unescaped_macro_pat = re.compile(r"([^%])%([^%])")
+
+def escape_macros(text):
+    escaped = unescaped_macro_pat.sub("\\1%%\\2", text)
+    return escaped
 
 def format_lines(lines):
     first = 1
@@ -105,7 +133,7 @@ def format_lines(lines):
     perexpr = re.compile(r"([^%])%([^%])")
     for line in lines:
         if line:
-            line = perexpr.sub("\\1%%\\2", line)
+            line = escape_macros(line)
             if first:
                 first = 0
                 line = line.lstrip()
@@ -130,8 +158,10 @@ class ChangelogByAuthor:
 
 def group_releases_by_author(releases):
     allauthors = []
+    grouped = []
     for release in releases:
         authors = {}
+        latest = None
         for revision in release.revisions:
             authors.setdefault(revision.author, []).append(revision)
 
@@ -144,60 +174,114 @@ def group_releases_by_author(releases):
             revdeco = [(r.revision, r) for r in revs]
             revdeco.sort(reverse=1)
             author.revisions = [t[1] for t in revdeco]
-            decorated.append((max(revdeco)[0], author))
+            revlatest = author.revisions[0]
+            # keep the latest revision even for silented authors (below)
+            if latest is None or revlatest.revision > latest.revision:
+                latest = revlatest
+            count = sum(len(rev.lines) for rev in author.revisions)
+            if count == 0:
+                # skipping author with only silented lines
+                continue
+            decorated.append((revdeco[0][0], author))
+
+        if not decorated:
+            # skipping release with only authors with silented lines
+            continue
 
         decorated.sort(reverse=1)
         release.authors = [t[1] for t in decorated]
-        # the difference between a released and a not released
-        # ChangelogRelease is the way the release numbers is obtained. So,
-        # when this is a released, we already have it, but if we don't, we
-        # should get de version/release string using getrelease and then
-        # get the first
+        # the difference between a released and a not released _Release is
+        # the way the release numbers is obtained. So, when this is a
+        # released, we already have it, but if we don't, we should get de
+        # version/release string using getrelease and then get the first
         first, release.authors = release.authors[0], release.authors[1:]
         release.author_name = first.name
         release.author_email = first.email
-        release.date = first.revisions[0].date
-        release.raw_date = first.revisions[0].raw_date
         release.release_revisions = first.revisions
-        release.revision = first.revisions[0].revision
 
-    return releases
-            
+        #release.date = first.revisions[0].date
+        release.date = latest.date
+        release.raw_date = latest.raw_date
+        #release.revision = first.revisions[0].revision
+        release.revision = latest.revision
+
+        grouped.append(release)
+
+    return grouped
+
+
+def group_revisions_by_author(currentlog):
+    revisions = []
+    last_author = None
+    for entry in currentlog:
+        revision = ChangelogRevision()
+        revision.lines = format_lines(entry.lines)
+        revision.raw_date = entry.date
+        revision.date = parse_raw_date(entry.date)
+        revision.revision = entry.revision
+        if entry.author == last_author:
+            revisions[-1].revisions.append(revision)
+        else:
+            author = ChangelogByAuthor()
+            author.name, author.email = get_author_name(entry.author)
+            author.revisions = [revision]
+            revisions.append(author)
+        last_author = entry.author
+    return revisions
+
 
 emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>")
 
+def get_author_name(author):
+    found = emailpat.match(config.get("users", author, author))
+    name = ((found and found.group("name")) or author)
+    email = ((found and found.group("email")) or author)
+    return name, email
+
+def parse_raw_date(rawdate):
+    return time.strftime("%a %b %d %Y", rawdate)
+
+def filter_log_lines(lines):
+    # lines in commit messages containing SILENT at any position will be
+    # skipped; commits with their log messages beggining with SILENT in the
+    # first positionj of the first line will have all lines ignored.
+    ignstr = config.get("log", "ignore-string", "SILENT")
+    if len(lines) and lines[0].startswith(ignstr):
+        return []
+    filtered = [line for line in lines if ignstr not in line]
+    return filtered
+
 
 def make_release(author=None, revision=None, date=None, lines=None,
         entries=[], released=True, version=None, release=None):
     rel = ChangelogRelease()
     rel.author = author
-    found = emailpat.match(config.get("users", author, author or ""))
-    rel.author_name = (found and found.group("name")) or author
-    rel.author_email = (found and found.group("email")) or author
+    if author:
+        rel.author_name, rel.author_email = get_author_name(author)
     rel.revision = revision
     rel.version = version
     rel.release = release
-    rel.date = (date and time.strftime("%a %b %d %Y", date)) or None
+    rel.date = (date and parse_raw_date(date)) or None
     rel.lines = lines
     rel.released = released
+    rel.visible = False
     for entry in entries:
+        lines = filter_log_lines(entry.lines)
+        if lines:
+            rel.visible = True
         revision = ChangelogRevision()
         revision.revision = entry.revision
-        revision.lines = format_lines(entry.lines)
-        revision.date = time.strftime("%a %b %d %Y", entry.date)
+        revision.lines = format_lines(lines)
+        revision.date = parse_raw_date(entry.date)
         revision.raw_date = entry.date
         revision.author = entry.author
-        found = emailpat.match(config.get("users", entry.author, entry.author))
-        revision.author_name = ((found and found.group("name")) or
-                entry.author)
-        revision.author_email = ((found and found.group("email")) or
-                entry.author)
+        (revision.author_name, revision.author_email) = \
+                get_author_name(entry.author)
         rel.revisions.append(revision)
     return rel
 
 
-def dump_file(releases, template=None):
-    
+def dump_file(releases, currentlog=None, template=None):
     templpath = template or config.get("template", "path", None)
     params = {}
     if templpath is None or not os.path.exists(templpath):
@@ -207,10 +291,12 @@ def dump_file(releases, template=None):
     else:
         params["file"] = templpath
     releases_author = group_releases_by_author(releases)
+    revisions_author = group_revisions_by_author(currentlog)
     params["searchList"] = [{"releases_by_author" : releases_author,
-                             "releases" : releases}]
+                             "releases" : releases,
+                             "revisions_by_author": revisions_author}]
     t = Template(**params)
-    return repr(t)
+    return t.respond()
 
 
 class InvalidEntryError(Exception):
@@ -249,8 +335,30 @@ def get_revision_offset():
                       "file(s).")
     return revoffset or 0
 
+oldmsgpat = re.compile(
+        r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.")
 
-def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, template=None):
+def parse_markrelease_log(relentry):
+    if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \
+            or parse_repsys_entry(relentry))):
+        raise InvalidEntryError
+    from_rev = None
+    path = None
+    for changed in relentry.changed:
+        if changed["action"] == "A" and changed["from_rev"]:
+            from_rev = changed["from_rev"]
+            path = changed["path"]
+            break
+    else:
+        raise InvalidEntryError
+    # get the version and release from the names in the path, do not relay
+    # on log messages
+    version, release = path.rsplit(os.path.sep, 3)[-2:]
+    return version, release, from_rev
+
+
+def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
+        template=None, macros=[]):
     size = size or 0
     concat = config.get("log", "concat", "").split()
     revoffset = get_revision_offset()
@@ -261,68 +369,74 @@ def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, template=None):
             strict_node_history=False, noerror=1)) or []
     currentlog = list(svn.log(pkgcurrenturl, 
             strict_node_history=False,
-            revision_start=SVN.revision(rev),
-            revision_end=SVN.revision(revoffset), limit=size))
-    lastauthor = None
-    previous_revision = 0
-    currelease = None
+            revision_start=SVN.makerev(rev),
+            revision_end=SVN.makerev(revoffset), limit=size))
+
+    # sort releases by copyfrom-revision, so that markreleases for same
+    # revisions won't look empty
+    releasesdata = []
+    if releaseslog:
+        for relentry in releaseslog[::-1]:
+            try:
+                (version, release, relrevision) = \
+                        parse_markrelease_log(relentry)
+            except InvalidEntryError:
+                continue
+            releasesdata.append((relrevision, -relentry.revision, relentry, 
+                version, release))
+        releasesdata.sort()
+
+    # collect valid releases using the versions provided by the changes and
+    # the packages
+    prevrevision = 0
     releases = []
-
-    # for the emergency bug fixer: the [].sort() is done using the 
-    # decorate-sort-undecorate pattern
-    releases_data = []
-    for relentry in releaseslog[::-1]:
-        try:
-            revinfo = parse_repsys_entry(relentry)
-        except InvalidEntryError:
+    for (relrevision, dummy, relentry, version, release) in releasesdata:
+        if prevrevision == relrevision: 
+            # ignore older markrelease of the same revision, since they
+            # will have no history
             continue
-        try:
-            release_number = int(revinfo["revision"])
-        except (KeyError, ValueError):
-            raise Error, "Error parsing data from log entry from r%s" % \
-                            relentry.revision  
-        releases_data.append((release_number, relentry, revinfo))
-    releases_data.sort()
-
-    for release_number, relentry, revinfo in releases_data:
-        # get entries newer than 'previous' and older than 'relentry'
         entries = [entry for entry in currentlog
-                    if release_number >= entry.revision and
-                      (previous_revision < entry.revision)]
+                    if relrevision >= entry.revision and
+                      (prevrevision < entry.revision)]
         if not entries:
             #XXX probably a forced release, without commits in current/,
-            # check if this is the right behavior and if some release is
-            # not being lost.
+            # check if this is the right behavior
+            sys.stderr.write("warning: skipping (possible) release "
+                    "%s-%s@%s, no commits since previous markrelease (r%r)\n" %
+                    (version, release, relrevision, prevrevision))
             continue
 
         release = make_release(author=relentry.author,
                 revision=relentry.revision, date=relentry.date,
                 lines=relentry.lines, entries=entries,
-                version=revinfo["version"], release=revinfo["release"])
+                version=version, release=release)
         releases.append(release)
-        previous_revision = release_number
-
+        prevrevision = relrevision
+            
     # look for commits that have been not submited (released) yet
     # this is done by getting all log entries newer (revision larger)
-    # than releaseslog[0]
-    latest_revision = releaseslog[0].revision
+    # than releaseslog[0] (in the case it exists)
+    if releaseslog:
+        latest_revision = releaseslog[0].revision
+    else:
+        latest_revision = 0
     notsubmitted = [entry for entry in currentlog 
                     if entry.revision > latest_revision]
     if notsubmitted:
         # if they are not submitted yet, what we have to do is to add
         # a release/version number from getrelease()
-        version, release = getrelease(pkgdirurl)
+        version, release = getrelease(pkgdirurl, macros=macros)
         toprelease = make_release(entries=notsubmitted, released=False,
                         version=version, release=release)
         releases.append(toprelease)
 
-    data = dump_file(releases[::-1], template=template)
+    data = dump_file(releases[::-1], currentlog=currentlog, template=template)
     return data
 
 
 
 def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
-        submit=False, template=None):
+        submit=False, template=None, macros=[]):
     newlines = []
     found = 0
     
@@ -339,7 +453,7 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
     # Create new changelog
     newlines.append("\n\n%changelog\n")
     newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
-        template=template))
+        template=template, macros=macros))
 
     # Merge old changelog, if available
     oldurl = config.get("log", "oldurl")
@@ -347,15 +461,19 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
         svn = SVN()
         tmpdir = tempfile.mktemp()
         try:
-            pkgname = os.path.basename(pkgdirurl)
+            pkgname = RepSysTree.pkgname(pkgdirurl)
             pkgoldurl = os.path.join(oldurl, pkgname)
             if svn.exists(pkgoldurl):
-                svn.export(pkgoldurl, tmpdir, rev=rev)
+                # we're using HEAD here because fixes in misc/ (oldurl) may
+                # be newer than packages' last changed revision.
+                svn.export(pkgoldurl, tmpdir)
                 logfile = os.path.join(tmpdir, "log")
                 if os.path.isfile(logfile):
                     file = open(logfile)
                     newlines.append("\n")
-                    newlines.append(file.read())
+                    log = file.read()
+                    log = escape_macros(log)
+                    newlines.append(log)
                     file.close()
         finally:
             if os.path.isdir(tmpdir):
diff --git a/RepSys/mirror.py b/RepSys/mirror.py
new file mode 100644
index 0000000..a24f594
--- /dev/null
+++ b/RepSys/mirror.py
@@ -0,0 +1,42 @@
+import os
+import urlparse
+
+from RepSys import config
+from RepSys.svn import SVN
+
+def relocate_path(oldparent, newparent, url):
+    subpath = url[len(oldparent)-1:]
+    newurl = newparent + "/" + subpath # subpath usually gets / at begining
+    return newurl
+
+def enabled():
+    mirror = config.get("global", "mirror")
+    default_parent = config.get("global", "default_parent")
+    return (mirror is not None and 
+            default_parent is not None)
+
+def mirror_relocate(oldparent, newparent, url, wcpath):
+    svn = SVN(noauth=True)
+    newurl = relocate_path(oldparent, newparent, url)
+    svn.switch(newurl, url, path=wcpath, relocate="True")
+    return newurl
+
+def switchto_parent(svn, url, path):
+    """Relocates the working copy to default_parent"""
+    mirror = config.get("global", "mirror")
+    default_parent = config.get("global", "default_parent")
+    newurl = mirror_relocate(mirror, default_parent, url, path)
+    return newurl
+
+def switchto_mirror(svn, url, path):
+    mirror = config.get("global", "mirror")
+    default_parent = config.get("global", "default_parent")
+    newurl = mirror_relocate(default_parent, mirror, url, path)
+    return newurl
+
+def checkout_url(url):
+    mirror = config.get("global", "mirror")
+    default_parent = config.get("global", "default_parent")
+    if mirror is not None and default_parent is not None:
+        return relocate_path(default_parent, mirror, url)
+    return url
diff --git a/RepSys/plugins/__init__.py b/RepSys/plugins/__init__.py
new file mode 100644
index 0000000..e4f4e08
--- /dev/null
+++ b/RepSys/plugins/__init__.py
@@ -0,0 +1,27 @@
+import os
+
+loaded = {}
+
+def load():
+    # based on smart's plugin system 
+    pluginsdir = os.path.dirname(__file__)
+    for entry in os.listdir(pluginsdir):
+        if entry != "__init__.py" and entry.endswith(".py"):
+            name = entry[:-3]
+            loaded[name] = __import__("RepSys.plugins."+name, {}, {},
+                    [name])
+        elif os.path.isdir(entry):
+            initfile = os.path.join(entry, "__init__.py")
+            if os.path.isfile(initfile):
+                loaded[entry] = __import__("RepSys.plugins."+entry, {}, {},
+                        [entry])
+
+def list():
+    return loaded.keys()
+
+def help(name):
+    from RepSys import Error
+    try:
+        return loaded[name].__doc__
+    except KeyError:
+        raise Error, "plugin %s not found" % name
diff --git a/RepSys/plugins/ldapusers.py b/RepSys/plugins/ldapusers.py
new file mode 100644
index 0000000..75d362c
--- /dev/null
+++ b/RepSys/plugins/ldapusers.py
@@ -0,0 +1,164 @@
+"""
+A Repsys plugin for obtaining users from a LDAP server.
+
+In order to enable the plugin, the user must define the following 
+options in the [global] section of repsys.conf:
+
+    ldap-server [required]
+        the host name of the LDAP server
+    ldap-port [optional] [default: 389]
+        the port of the LDAP server
+    ldap-base [required]
+        the base DN where the search will be performed
+    ldap-binddn [optional] [default: empty]
+        the DN used to bind
+    ldap-bindpw [optional] [default: empty]
+        the password used to bind
+    ldap-filterformat [optional] 
+            [default: (&(objectClass=inetOrgPerson)(uid=$username))]
+        RFC-2254 filter string used in the search of the user entry.
+        Note that this is a python template string and will have the 
+        user name as parameter. For example:
+
+           ldap-filterformat = (&(objectClass=inetOrgPerson)(uid=$username))
+
+        Will result in the search filter:
+
+           (&(objectClass=inetOrgPerson)(uid=john))
+
+    ldap-resultformat [optional] [default: $cn <$mail>]
+        This is a python template string. This string will be 
+        formatted using one dict object containing the fields
+        returned in the LDAP search, for example:
+
+          >>> format = Template("$cn <$mail>")
+          >>> d = search(basedn, filter)
+          >>> d
+          {"cn": "John Doe", "mail": "john@mandriva.org", 
+           "uidNumber": "1290", "loginShell": "/bin/bash", 
+            ... many other attributes ... }
+          >>> value = format.substitute(d)
+          >>> print value
+          John Doe <john@mandriva.org>
+
+        Note that only the first value of the attributes will be 
+        used.
+
+When the searched option is not found, it will try in repsys.conf. All
+the values found.  (including from repsys.conf) will be cached between
+each configuration access.
+
+This plugin requires the package python-ldap.
+
+For more information, look http://qa.mandriva.com/show_bug.cgi?id=30549
+"""
+from RepSys import Error, config
+
+import string
+
+users_cache = {}
+
+class LDAPError(Error):
+    def __init__(self, ldaperr):
+        self.ldaperr = ldaperr
+        name = ldaperr.__class__.__name__
+        desc = ldaperr.message["desc"]
+        self.message = "LDAP error %s: %s" % (name, desc)
+        self.args = self.message,
+
+def strip_entry(entry):
+    "Leave only the first value in all keys in the entry"
+    new = dict((key, value[0]) for key, value in entry.iteritems())
+    return new
+
+def interpolate(optname, format, data):
+    tmpl = string.Template(format)
+    try:
+        return tmpl.substitute(data)
+    except KeyError, e:
+        raise Error, "the key %s was not found in LDAP search, " \
+                "check your %s configuration" % (e, optname)
+    except (TypeError, ValueError), e:
+        raise Error, "LDAP response formatting error: %s. Check " \
+                "your %s configuration" % (e, optname)
+
+def used_attributes(format):
+    class DummyDict:
+        def __init__(self):
+            self.found = []
+        def __getitem__(self, key):
+            self.found.append(key)
+            return key
+    dd = DummyDict()
+    t = string.Template(format)
+    t.safe_substitute(dd)
+    return dd.found
+
+def make_handler():
+    server = config.get("global", "ldap-server")
+    try:
+        port = int(config.get("global", "ldap-port", 389))
+    except ValueError:
+        raise Error, "the option ldap-port requires an integer, please "\
+                "check your configuration files"
+    basedn = config.get("global", "ldap-base")
+    binddn = config.get("global", "ldap-binddn")
+    bindpw = config.get("global", "ldap-bindpw", "")
+    filterformat = config.get("global", "ldap-filterformat",
+            "(&(objectClass=inetOrgPerson)(uid=$username))", raw=1)
+    format = config.get("global", "ldap-resultformat", "$cn <$mail>", raw=1)
+
+    if server is None:
+        def dummy_wrapper(section, option=None, default=None, walk=False):
+            return config.get(section, option, default, wrap=False)
+        return dummy_wrapper
+
+    try:
+        import ldap
+    except ImportError:
+        raise Error, "LDAP support needs the python-ldap package "\
+                "to be installed"
+    else:
+        from ldap.filter import escape_filter_chars
+
+    def users_wrapper(section, option=None, default=None, walk=False):
+        global users_cache
+        if walk:
+            raise Error, "ldapusers plugin does not support user listing"
+        assert option is not None, \
+                "When not section walking, option is required"
+
+        value = users_cache.get(option)
+        if value is not None:
+            return value
+
+        try:
+            l = ldap.open(server, port)
+            if binddn:
+                l.bind(binddn, bindpw)
+        except ldap.LDAPError, e:
+            raise LDAPError(e)
+        try:
+            data = {"username": escape_filter_chars(option)}
+            filter = interpolate("ldap-filterformat", filterformat, data)
+            attrs = used_attributes(format)
+            try:
+                found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter,
+                        attrlist=attrs)
+            except ldap.LDAPError, e:
+                raise LDAPError(e)
+            if found:
+                dn, entry = found[0]
+                entry = strip_entry(entry)
+                value = interpolate("ldap-resultformat", format, entry)
+            else:
+                # issue a warning?
+                value = config.get(section, option, default, wrap=False)
+            users_cache[option] = value
+            return value
+        finally:
+            l.unbind_s()
+
+    return users_wrapper
+
+config.wrap("users", handler=make_handler())
diff --git a/RepSys/plugins/sample.py.txt b/RepSys/plugins/sample.py.txt
new file mode 100644
index 0000000..9877f3c
--- /dev/null
+++ b/RepSys/plugins/sample.py.txt
@@ -0,0 +1,14 @@
+# Sample repsys plugin. In order to test it, rename to sample.py
+# vim:ft=python
+from RepSys import config
+
+def users_wrapper(section, option=None, default=None, walk=False):
+    d = {"foolano": "Foolano De Tal <foolano@bla.com>",
+         "ceeclano": "Ceeclano Algumacoisa <ceeclano@bli.com>",
+         "beltrano": "Beltrano Bla <beltrano@mail.ru>"}
+    if walk:
+        return d.items()
+    
+    return d.get(option, default)
+
+config.wrap("users", handler=users_wrapper)
diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py
index 8b8c8b7..39e78fc 100644
--- a/RepSys/rpmutil.py
+++ b/RepSys/rpmutil.py
@@ -1,10 +1,12 @@
 #!/usr/bin/python
-from RepSys import Error, config
+from RepSys import Error, config, RepSysTree
+from RepSys import mirror
 from RepSys.svn import SVN
-from RepSys.rpm import SRPM
+from RepSys.simplerpm import SRPM
 from RepSys.log import specfile_svn2rpm
 from RepSys.util import execcmd
 import pysvn
+import rpm
 import tempfile
 import shutil
 import glob
@@ -26,6 +28,11 @@ def get_spec(pkgdirurl, targetdir=".", submit=False):
         if os.path.isdir(tmpdir):
             shutil.rmtree(tmpdir)
 
+def rpm_macros_defs(macros):
+    defs = ("--define \"%s %s\"" % macro for macro in macros)
+    args = " ".join(defs)
+    return args
+
 def get_srpm(pkgdirurl,
              mode = "current",
              targetdirs = None,
@@ -38,6 +45,7 @@ def get_srpm(pkgdirurl,
              scripts = [], 
              submit = False,
              template = None,
+             macros = [],
              verbose = 0):
     svn = SVN()
     tmpdir = tempfile.mktemp()
@@ -58,7 +66,7 @@ def get_srpm(pkgdirurl,
             geturl = os.path.join(pkgdirurl, "current")
         else:
             raise Error, "unsupported get_srpm mode: %s" % mode
-        svn.checkout(geturl, tmpdir, revision=SVN.revision(revision))
+        svn.export(geturl, tmpdir, revision=SVN.makerev(revision))
         srpmsdir = os.path.join(tmpdir, "SRPMS")
         os.mkdir(srpmsdir)
         specsdir = os.path.join(tmpdir, "SPECS")
@@ -69,9 +77,12 @@ def get_srpm(pkgdirurl,
         if svnlog:
             submit = not not revision
             specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
-                    template=template)
-        revisionreal = svn.info(tmpdir).revision.number
+                    template=template, macros=macros)
+        #FIXME revisioreal not needed if revision is None
+        #FIXME use geturl instead of pkgdirurl
+        revisionreal = svn.revision(pkgdirurl)
         for script in scripts:
+            #FIXME revision can be "None"
             status, output = execcmd(script, tmpdir, spec, str(revision),
                                      noerror=1)
             if status != 0:
@@ -79,11 +90,13 @@ def get_srpm(pkgdirurl,
         if packager:
             packager = " --define 'packager %s'" % packager
 
-        execcmd("rpm -bs --nodeps %s %s %s %s %s %s %s %s %s" %
+        defs = rpm_macros_defs(macros)
+        execcmd("rpm -bs --nodeps %s %s %s %s %s %s %s %s %s %s" %
             (topdir, builddir, rpmdir, sourcedir, specdir, 
-             srcrpmdir, patchdir, packager, spec))
+             srcrpmdir, patchdir, packager, spec, defs))
 
         if revision and revisionreal:
+            #FIXME duplicate glob line
             srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0]
             srpminfo = SRPM(srpm)
             release = srpminfo.release
@@ -94,7 +107,8 @@ def get_srpm(pkgdirurl,
             targetdirs = (".",)
         targetsrpms = []
         for targetdir in targetdirs:
-            targetsrpm = os.path.join(os.path.realpath(targetdir), os.path.basename(srpm))
+            targetsrpm = os.path.join(os.path.realpath(targetdir), 
+                    os.path.basename(srpm))
             targetsrpms.append(targetsrpm)
             if verbose:
                 sys.stderr.write("Wrote: %s\n" %  targetsrpm)
@@ -226,7 +240,7 @@ def create_package(pkgdirurl, log="", verbose=0):
     svn = SVN()
     tmpdir = tempfile.mktemp()
     try:
-        basename = os.path.basename(pkgdirurl)
+        basename = RepSysTree.pkgname(pkgdirurl)
         if verbose:
             print "Creating package directory...",
         sys.stdout.flush()
@@ -269,7 +283,7 @@ def mark_release(pkgdirurl, version, release, revision):
     versionurl = "/".join([releasesurl, version])
     releaseurl = "/".join([versionurl, release])
     if svn.exists(releaseurl, noerror=1):
-        raise cncrep.Error, "release already exists"
+        raise Error, "release already exists"
     svn.mkdir(releasesurl, noerror=1,
               log="Created releases directory.")
     svn.mkdir(versionurl, noerror=1,
@@ -282,13 +296,13 @@ def mark_release(pkgdirurl, version, release, revision):
              log="Copying release %s-%s to pristine/ directory." %
                  (version, release))
     markreleaselog = create_markrelease_log(version, release, revision)
-    svn.copy(currenturl, releaseurl, rev=revision,
+    svn.copy(currenturl, releaseurl, revision=revision,
              log=markreleaselog)
 
-def check_changed(url, all=0, show=0, verbose=0):
+def check_changed(pkgdirurl, all=0, show=0, verbose=0):
     svn = SVN()
     if all:
-        baseurl = url
+        baseurl = pkgdirurl
         packages = []
         if verbose:
             print "Getting list of packages...",
@@ -299,7 +313,7 @@ def check_changed(url, all=0, show=0, verbose=0):
         if not packages:
             raise Error, "couldn't get list of packages"
     else:
-        baseurl, basename = os.path.split(url)
+        baseurl, basename = os.path.split(pkgdirurl)
         packages = [basename]
     clean = []
     changed = []
@@ -332,22 +346,103 @@ def check_changed(url, all=0, show=0, verbose=0):
                 if verbose:
                     print "clean"
                 clean.append(package)
-    if verbose and all:
-        print "Total clean packages: %s" % len(clean)
-        print "Total CHANGED packages: %d" % len(changed)
-        print "Total NO CURRENT packages: %s" % len(nocurrent)
-        print "Total NO PRISTINE packages: %s" % len(nopristine)
+    if verbose:
+        if not packages:
+            print "No packages found!"
+        elif all:
+            print "Total clean packages: %s" % len(clean)
+            print "Total CHANGED packages: %d" % len(changed)
+            print "Total NO CURRENT packages: %s" % len(nocurrent)
+            print "Total NO PRISTINE packages: %s" % len(nopristine)
     return {"clean": clean,
             "changed": changed,
             "nocurrent": nocurrent,
             "nopristine": nopristine}
 
-def checkout(url, path=None, revision=None):
+def checkout(pkgdirurl, path=None, revision=None):
     svn = SVN()
-    current = os.path.join(url, "current")
+    current = os.path.join(pkgdirurl, "current")
     if path is None:
-        _, path = os.path.split(url)
-    svn.checkout(current, path, revision=SVN.revision(revision), show=1)
+        _, path = os.path.split(pkgdirurl)
+    if mirror.enabled():
+        current = mirror.checkout_url(current)
+        print "checking out from mirror", current
+    svn.checkout(current, path, revision=SVN.makerev(revision), show=1)
+
+def sync(dryrun=False):
+    svn = SVN()
+    cwd = os.getcwd()
+    dirname = os.path.basename(cwd)
+    if dirname == "SPECS" or dirname == "SOURCES":
+        topdir = os.pardir
+    else:
+        topdir = ""
+    # run svn info because svn st does not complain when topdir is not an
+    # working copy
+    svn.info(topdir or ".")
+    specsdir = os.path.join(topdir, "SPECS/")
+    sourcesdir = os.path.join(topdir, "SOURCES/")
+    for path in (specsdir, sourcesdir):
+        if not os.path.isdir(path):
+            raise Error, "%s directory not found" % path
+    specs = glob.glob(os.path.join(specsdir, "*.spec"))
+    if not specs:
+        raise Error, "no .spec files found in %s" % specsdir
+    specpath = specs[0] # FIXME better way?
+    try:
+        spec = rpm.TransactionSet().parseSpec(specpath)
+    except rpm.error, e:
+        raise Error, "could not load spec file: %s" % e
+    sources = [os.path.basename(name)
+            for name, no, flags in spec.sources()]
+    sourcesst = dict((os.path.basename(st.path), st)
+            for st in svn.status(sourcesdir, get_all=False, ignore=True))
+    toadd = []
+    for source in sources:
+        sourcepath = os.path.join(sourcesdir, source)
+        if sourcesst.get(source):
+            if os.path.isfile(sourcepath):
+                toadd.append(sourcepath)
+            else:
+                sys.stderr.write("warning: %s not found\n" % sourcepath)
+    # rm entries not found in sources and still in svn
+    found = os.listdir(sourcesdir)
+    toremove = []
+    for entry in found:
+        if entry == ".svn":
+            continue
+        status = sourcesst.get(entry)
+        if status is None and entry not in sources:
+            path = os.path.join(sourcesdir, entry)
+            toremove.append(path)
+    for path in toremove:
+        print "D\t%s" % path
+        if not dryrun:
+            svn.remove(path)
+    for path in toadd:
+        print "A\t%s" % path
+        if not dryrun:
+            svn.add(path)
+
+def commit(target=".", message=None):
+    svn = SVN()
+    info = svn.info(target)
+    url = info.url
+    if url is None:
+        raise Error, "working copy URL not provided by svn info"
+    if mirror.enabled():
+        newurl = mirror.switchto_parent(svn, url, target)
+        print "relocated to", newurl
+    try:
+        # we can't use the svn object here because pexpect hides VISUAL
+        mopt = ""
+        if message is not None:
+            mopt = "-m \"%s\"" % message
+        os.system("svn ci %s %s" % (mopt, target))
+    finally:
+        if mirror.enabled():
+            mirror.switchto_mirror(svn, newurl, target)
+            print "relocated back to", url
 
 def get_submit_info(path):
     path = os.path.abspath(path)
diff --git a/RepSys/svn.py b/RepSys/svn.py
index f055d8a..2fe9ad1 100644
--- a/RepSys/svn.py
+++ b/RepSys/svn.py
@@ -19,6 +19,7 @@ class SVNLogEntry:
         self.revision = revision
         self.author = author
         self.date = date
+        self.changed = []
         self.lines = []
 
     def __cmp__(self, other):
@@ -33,25 +34,42 @@ class SVN:
         self._client = pysvn.Client()
         self._client_lock = threading.Lock()
         self._client.callback_get_log_message = self._log_handler
+        self._client.callback_get_login = self._unsupported_auth
+        self._client.callback_ssl_client_cert_password_prompt = \
+                self._unsupported_auth
+        self._client.callback_ssl_client_cert_prompt = \
+                self._unsupported_auth
+        self._client.callback_ssl_server_prompt = \
+                self._unsupported_auth
 
     def _log_handler(self):
         if self._current_message is None:
+            #TODO make it use EDITOR
             raise ValueError, "No log message defined"
         return True, self._current_message
 
+    def _unsupported_auth(self, *args, **kwargs):
+        raise SVNError, "svn is trying to get login information, " \
+                "seems that you're not using ssh-agent"
+
     def _get_log_message(self, received_kwargs):
         message = received_kwargs.pop("log", None)
         messagefile = received_kwargs.pop("logfile", None)
         if messagefile and not message:
             message = open(messagefile).read()
         return message
-            
-    def _make_wrapper(self, meth):
+
+    def _set_notify_callback(self, callback):
+        self._client.callback_notify = callback
+
+    def _make_wrapper(self, meth, notify=None):
         def wrapper(*args, **kwargs):
             self._client_lock.acquire()
             try:
                 self._current_message = self._get_log_message(kwargs)
                 ignore_errors = kwargs.pop("noerror", None)
+                if notify:
+                    self._client.callback_notify = notify
                 try:
                     return meth(*args, **kwargs)
                 except pysvn.ClientError, (msg,):
@@ -59,41 +77,63 @@ class SVN:
                         raise SVNError, msg
                     return None
             finally:
-                self._current_message = None
                 self._client_lock.release()
+                self._current_message = None
         return wrapper
 
-    def __getattr__(self, attrname):
+    def _client_wrap(self, attrname):
         meth = getattr(self._client, attrname)
         wrapper = self._make_wrapper(meth)
         return wrapper
 
-    def revision(number=None, head=None):
+    def __getattr__(self, attrname):
+        return self._client_wrap(attrname)
+    
+    def makerev(number=None, head=None):
         if number is not None:
             args = (pysvn.opt_revision_kind.number, number)
         else:
             args = (pysvn.opt_revision_kind.head,)
         return pysvn.Revision(*args)
-    revision = staticmethod(revision)
+    makerev = staticmethod(makerev)
+
+    def revision(self, url):
+        infos = self._client.info2(url, recurse=False)
+        revnum= infos[0][1].rev.number
+        return revnum
 
     # this override method fixed the problem in pysvn's mkdir which
     # requires a log_message parameter
     def mkdir(self, path, log=None, **kwargs):
-        meth = self.__getattr__("mkdir")
+        meth = self._client_wrap("mkdir")
         # we can't raise an error because pysvn's mkdir will use
         # log_message only if path is remote, but it *always* requires this
         # parameter. Also, 'log' is never used.
         log = log or "There's a silent bug in your code"
         return meth(path, log, log=None, **kwargs)
+    
+    def checkout(self, url, targetpath, show=False, **kwargs):
+        if show:
+            def callback(event):
+                types = pysvn.wc_notify_action
+                action = event["action"]
+                if action == types.update_add:
+                    print "A    %s" % event["path"]
+                elif action == types.update_completed:
+                    print "Checked out revision %d" % \
+                            event["revision"].number
+            self._set_notify_callback(callback)
+        meth = self._client_wrap("checkout")
+        meth(url, targetpath, **kwargs)
 
     def checkin(self, path, log, **kwargs):
         # XXX use EDITOR when log empty
-        meth = self.__getattr__("checkin")
+        meth = self._client_wrap("checkin")
         return meth(path, log, log=None, **kwargs)
-
+    
     def log(self, *args, **kwargs):
-        meth = self.__getattr__("log")
-        entries = meth(*args, **kwargs)
+        meth = self._client_wrap("log")
+        entries = meth(discover_changed_paths=True, *args, **kwargs)
         if entries is None:
             return
         for entrydic in entries:
@@ -101,11 +141,32 @@ class SVN:
                                 entrydic["author"],
                                 time.localtime(entrydic["date"]))
             entry.lines[:] = entrydic["message"].split("\n")
+            for cp in entrydic["changed_paths"]:
+                from_rev = cp["copyfrom_revision"]
+                if from_rev:
+                    from_rev = from_rev.number
+                changed = {
+                    "action": cp["action"],
+                    "path": cp["path"],
+                    "from_rev": from_rev,
+                    "from_path": cp["copyfrom_path"],
+                }
+                entry.changed.append(changed)
             yield entry
-
+    
     def exists(self, path):
         return self.ls(path, noerror=1) is not None
 
+    def diff(self, *args, **kwargs):
+        head = pysvn.Revision(pysvn.opt_revision_kind.head)
+        revision1 = kwargs.pop("revision1", head)
+        revision2 = kwargs.pop("revision2", head)
+        tmpdir = tempfile.gettempdir()
+        meth = self._client_wrap("diff")
+        diff_text = meth(tmpdir, revision1=revision1, revision2=revision2,
+                *args, **kwargs)
+        return diff_text
+
     def _edit_message(self, message):
         # argh!
         editor = os.getenv("EDITOR", "vim")
@@ -116,8 +177,10 @@ class SVN:
             f.write(message)
             f.close()
             lastchange = os.stat(fpath).st_mtime
-            while 1:
-                os.system("%s %s" % (editor, fpath))
+            for i in xrange(10):
+                status = os.system("%s %s" % (editor, fpath))
+                if status != 0:
+                    raise SVNError, "the editor failed with %d" % status
                 newchange = os.stat(fpath).st_mtime
                 if newchange == lastchange:
                     print "Log message unchanged or not specified"
@@ -137,7 +200,7 @@ class SVN:
         return result
 
     def propedit(self, propname, pkgdirurl, revision, revprop=False):
-        revision = self.revision(revision)
+        revision = (revision)
         if revprop:
             propget = self.revpropget
             propset = self.revpropset
diff --git a/TODO.LDAP b/TODO.LDAP
new file mode 100644
index 0000000..54d6350
--- /dev/null
+++ b/TODO.LDAP
@@ -0,0 +1,4 @@
+- we should have a generic fqdn here to use round-robin DNS
+  enhancement for repsys: support multiple ldap servers here
+ (from repsys.conf in kenobi)
+- support STARTTLS 
diff --git a/create-srpm b/create-srpm
index 544b011..3dab068 100755
--- a/create-srpm
+++ b/create-srpm
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-from RepSys import Error, config
+from RepSys import Error, config, plugins
 from RepSys.rpmutil import get_srpm
 from RepSys.cgiutil import get_targets
 from RepSys.util import mapurl, execcmd, get_helper
@@ -52,7 +52,8 @@ class CmdIface:
                      packager=packager,
                      revname=1,
                      svnlog=1,
-                     scripts=target.scripts)
+                     scripts=target.scripts, 
+                     macros=target.macros)
 
             uploadsrpm = get_helper("upload-srpm")
             if uploadsrpm:
@@ -63,8 +64,12 @@ class CmdIface:
                         upload_command.append(x)
                 upload_command.append(targetname)
                 upload_command.append(targetsrpms[0])
-                status, output = execcmd(" ".join(upload_command))
-                os.unlink(targetsrpms[0])
+                status, output = execcmd(" ".join(upload_command),
+                        noerror=1)
+                if os.path.isfile(targetsrpms[0]):
+                    os.unlink(targetsrpms[0])
+                else:
+                    sys.stderr.write("warning: upload ok; temp file '%s' removed unexpectedly\n" % (targetsrpms[0]))
                 if status != 0:
                     raise CmdError, "Failed to upload %s:\n%s" % (packageurl, output)
         return 1
@@ -91,9 +96,14 @@ def parse_options():
 
 
 def main():
+    plugins.load()
     iface = CmdIface()
     opts, args = parse_options()
-    iface.submit_package(args[0], opts.revision, opts.target, opts.urlmap, opts.define)
+    try:
+        iface.submit_package(args[0], opts.revision, opts.target, opts.urlmap, opts.define)
+    except Error, e:
+        sys.stderr.write("error: %s\n" % str(e))
+        sys.exit(1)
     
 
 if __name__ == "__main__":
diff --git a/default.chlog b/default.chlog
index 4b767cf..aff3958 100644
--- a/default.chlog
+++ b/default.chlog
@@ -3,8 +3,11 @@
 #for $rel in $releases_by_author
 * $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
 + Revision: $rel.revision
- #if not $rel.released
-+ Status: not released
+## #if not $rel.released
+##+ Status: not released
+## #end if
+ #if not $rel.visible
++ rebuild (emptylog)
  #end if
  #for $rev in $rel.release_revisions
   #for $line in $rev.lines
@@ -13,6 +16,13 @@ $line
  #end for
 
  #for $author in $rel.authors
+  #if $author.revisions and not $author.revisions[0].lines
+    #continue
+  #end if
+  ##alternatively, one could use:
+  ###if $author.email == "root"
+  ## #continue
+  ###end if
   + $author.name <$author.email>
   #for $rev in $author.revisions
     #for $line in $rev.lines
diff --git a/repsys b/repsys
index e800a7f..804ad05 100755
--- a/repsys
+++ b/repsys
@@ -1,18 +1,21 @@
 #!/usr/bin/python
-from RepSys import Error
+from RepSys import Error, plugins
 from RepSys.command import *
 import getopt
 import sys
+import codecs
+import locale
 
 
 VERSION="1.7.r%s" % ("$Rev$".split()[-2].strip())
 
 HELP = """\
-$Id$
 Usage: repsys COMMAND [COMMAND ARGUMENTS]
 
 Useful commands:
     co
+    sync
+    ci
     submit
     create
     getspec
@@ -20,17 +23,36 @@ Useful commands:
     rpmlog
     changed
     authoremail
+    putsrpm
 
 Run "repsys COMMAND --help" for more information.
 
+Run "repsys --help-plugins" for help on loaded plugins.
+
 Written by Gustavo Niemeyer <gustavo@niemeyer.net>
 """
 
+def plugin_help(opt, val, parser, mode):
+    if parser is None:
+        prog = sys.argv[0]
+        print "Use %s --help-plugin <plugin name>" % prog
+        print "Available plugins:"
+        print
+        for name in plugins.list():
+            print name
+    else:
+        print plugins.help(parser)
+    raise SystemExit
+
 def parse_options():
     parser = OptionParser(help=HELP, version="%prog "+VERSION)
     parser.disable_interspersed_args()
     parser.add_option("--debug", action="store_true")
+    parser.add_option("--help-plugins", action="callback", callback=plugin_help)
+    parser.add_option("--help-plugin", type="string", dest="__ignore",
+            action="callback", callback=plugin_help)
     opts, args = parser.parse_args()
+    del opts.__ignore
     if len(args) < 1:
         parser.print_help(sys.stderr)
         sys.exit(1)
@@ -53,6 +75,14 @@ def dispatch_command(command, argv, debug=0):
     command_module.main()
     
 if __name__ == "__main__":
+    try:
+        plugins.load()
+    except Error, e:
+        sys.stderr.write("plugin initialization error: %s\n" % e)
+        sys.exit(1)
+    encoding = locale.getpreferredencoding()
+    sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors="replace")
+    sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors="replace")
     do_command(parse_options, dispatch_command)
 
 # vim:et:ts=4:sw=4
diff --git a/repsys.conf b/repsys.conf
index 5679c87..bbe99af 100644
--- a/repsys.conf
+++ b/repsys.conf
@@ -2,6 +2,8 @@
 verbose = no
 default_parent = svn+ssh://svn.mandriva.com/svn/packages/cooker
 url-map = svn\+ssh://svn\.mandriva\.com/(.*) file:///\1
+#mirror = http://svn.mandriva.com/svn/packages/cooker/
+#tempdir = /tmp
 
 [log]
 oldurl = svn+ssh://svn.mandriva.com/svn/packages/misc
@@ -9,27 +11,42 @@ oldurl = svn+ssh://svn.mandriva.com/svn/packages/misc
 # will be constructed (default zero, i.e., oldest
 # commit)
 revision-offset = 0
+# commit lines containing this string won't be shown in the changelog:
+ignore-string = SILENT
 
 [template]
 path = /usr/share/repsys/default.chlog
 
 [helper]
 create-srpm = /usr/share/repsys/create-srpm
+upload-srpm = /usr/local/bin/youri.devel
 
 [users]
-andreas = Andreas Hasenack <andreas@mandriva.com>
-boiko = Gustavo Pichorim Boiko <boiko@mandriva.com>
-cavassin = Wanderlei Cavassin <cavassin@mandriva.com>
-fcrozat = Frederic Crozat <fcrozat@mandriva.com>
-flepied = Frederic Lepied <flepied@mandriva.com>
-helio = Helio Chissini de Castro <helio@mandriva.com>
-lmontel = Laurent Montel <lmontel@mandriva.com>
-oden = Oden Eriksson <oeriksson@mandriva.com>
+# jsmith = John Smith <jsmith26@example.com>
 
 [submit]
+host = kenobi.mandriva.com
 default = Cooker
 
 [submit Cooker]
 target = /export/home/repsys
 allowed = svn+ssh://svn.mandriva.com/svn/packages/cooker
 scripts = /usr/share/repsys/rebrand-mdk
+##
+## rpm-macros refers to the sections containing the macros used for this
+## target. The values will be used to build the rpmbuild command line. For
+## example:
+## 
+##   [macros cooker]
+##   a = b
+##   c = %a
+##
+## will render in the command line: --define "a b" --define "c %a".
+##
+#rpm-macros = global cooker
+
+#[macros global]
+#distsuffix = mdv
+
+#[macros cooker]
+#mandriva_release = 2007.1