Initial work on the support of the binary files repository

It introduces a command "upload" to explicitly upload tarballs to the
repository and update the list on svn (the file SOURCES/sources). Also a
"del" command, to remove the reference to the file in the sources file. An
another "up", to pull possibly new tarballs from the binaries repository.

The command sync will automatically mark the new files that should go the
binaries repository. They should be uploaded with ci.

With the current code, packages can have tarballs gradually migrated as
tarballs of newer versions will appear.

In this implementation markrelease is required to be run in the same
machine of the repository (or at least should have direct fs access to it),
as it works by performing plain "cp"

11 files changed, 694 insertions, 33 deletions

diff --git a/CHANGES b/CHANGES
index 43bb51d..3faa796 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,7 @@
-* 1.6.20
+* Development version:
+- added support to "binary repository"
+- added new commands upload, up and del to help handling tarballs in the
+  binaries repository
 - dropped all authenticated access support: subversion authentication has
   been broken for a long time and the workarounds weren't decent. It will
   be back in 1.7.x.
diff --git a/RepSys/binrepo.py b/RepSys/binrepo.py
new file mode 100644
index 0000000..9fbd19e
--- /dev/null
+++ b/RepSys/binrepo.py
@@ -0,0 +1,425 @@
+from RepSys import Error, RepSysTree, config
+from RepSys.util import execcmd
+from RepSys.svn import SVN
+from RepSys.mirror import same_base
+
+import os
+import string
+import stat
+import sha
+import shutil
+import re
+import tempfile
+import urlparse
+from cStringIO import StringIO
+
+#TODO logging for markrelease
+
+DEFAULT_TARBALLS_REPO = "/tarballs"
+BINLIST_PENDING = "repsys-upload"
+BINLIST_DELETE = "repsys-delete"
+
+class ChecksumError(Error):
+    pass
+
+def copy_rsync(sources, dest, sourcehost=None, desthost=None,
+        archive=False, recurse=False):
+    """Simple inteface for rsync"""
+    args = ["rsync", "-i", "--log-format=\"%i %n\""]
+    if archive:
+        args.append("-a")
+    if recurse:
+        args.append("-r")
+    if sourcehost:
+        # "svn.mandriva.com:/foo/a /foo/b"
+        #TODO space escaping needed for sources
+        args.append("\"" + sourcehost + ":" + " ".join(sources) + "\"")
+    else:
+        args.extend(sources)
+    if desthost:
+        args.append(desthost + ":" + dest)
+    else:
+        args.append(dest)
+    execcmd(show=True, *args)
+
+def makedirs_remote(path, host):
+    tmpdir = tempfile.mkdtemp(prefix="repsys-makedirs")
+    try:
+        newpath = os.path.normpath(tmpdir + "/" + path)
+        os.makedirs(newpath)
+        copy_rsync(sources=[tmpdir + "/"], dest="/", desthost=host, recurse=True)
+    finally:
+        if os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
+
+def copy(sources, dest, sourcehost=None, desthost=None, makedirs=False,
+        hardlink=False):
+    """rsync-like copy
+
+    Note that a copy between two dirs will result in overwriting the
+    latter.
+    """
+    if desthost is None:
+        # we only need dest to contain the host name
+        try:
+            desthost, dpath = dest.split(":", 1)
+        except ValueError:
+            dpath = dest
+    if makedirs:
+        if desthost:
+            makedirs_remote(dpath, desthost)
+        else:
+            try:
+                os.makedirs(dpath)
+            except OSError, e:
+                if e.errno != 17: # already exists
+                    raise
+    if sourcehost or desthost:
+        copy_rsync(sources=sources, sourcehost=sourcehost, dest=dpath,
+                desthost=desthost, recurse=True, archive=True)
+    else:
+        opts = "-a"
+        if hardlink:
+            opts += "l"
+        for source in sources:
+            if os.path.isdir(source) and os.path.exists(dpath):
+                #FIXME ugly workaround to behave consistently between
+                # remote and local copies:
+                try:
+                    os.rmdir(dpath)
+                except OSError, e:
+                    raise Error, "can't overwrite directory: %s" % e
+            execcmd("cp %s %s %s" % (opts, source, dpath))
+
+def svn_basedir(target):
+    svn = SVN()
+    info = svn.info2(target)
+    if info is None:
+        # unversioned resource
+        newtarget = os.path.dirname(target)
+        info = svn.info2(newtarget)
+        assert info is not None, "svn_basedir should not be used with a "\
+                "non-versioned directory"
+    root = info["Repository Root"]
+    url = info["URL"]
+    kind = info["Node Kind"]
+    path = url[len(root):]
+    if kind == "directory":
+        return path
+    return os.path.dirname(path)
+
+def svn_root(target):
+    svn = SVN()
+    info = svn.info2(target)
+    if info is None:
+        newtarget = os.path.dirname(target)
+        info = svn.info2(newtarget)
+        assert info is not None
+    return info["Repository Root"]
+
+def enabled(url):
+    use = config.getbool("global", "use-binaries-repository", False)
+    default_parent = config.get("global", "default_parent", None)
+    if url and use and default_parent and same_base(url, default_parent):
+        return True
+    return False
+
+def target_url(path=None):
+    from RepSys.rpmutil import get_submit_info
+    base = config.get("global", "binaries-repository", None)
+    if base is None:
+        default_parent = config.get("global", "default_parent", None)
+        if default_parent is None:
+            raise Error, "no binaries-repository nor default_parent "\
+                    "configured"
+        comps = urlparse.urlparse(default_parent)
+        base = comps[1] + ":" + DEFAULT_TARBALLS_REPO
+    if path:
+        target = os.path.normpath(base + "/" + svn_basedir(path))
+    else:
+        target = base
+    return target
+
+def file_hash(path):
+    sum = sha.new()
+    f = open(path)
+    while True:
+        block = f.read(4096)
+        if not block:
+            break
+        sum.update(block)
+    f.close()
+    return sum.hexdigest()
+
+def check_hash(path, sum):
+    newsum = file_hash(path)
+    if newsum != sum:
+        raise ChecksumError, "different checksums for %s: expected %s, "\
+                "but %s was found" % (path, sum, newsum)
+
+def parse_sources_stream(stream):
+    entries = {}
+    for rawline in stream:
+        line = rawline.strip()
+        try:
+            sum, name = line.split(None, 1)
+        except ValueError:
+            # failed to unpack, line format error
+            raise Error, "invalid line in sources file: %s" % rawline
+        entries[name] = sum
+    return entries
+
+def parse_sources(path, force=False):
+    if not os.path.exists(path) and not force:
+        return {}
+    basedir = os.path.dirname(path)
+    spath = os.path.join(basedir, "sources")
+    f = open(spath)
+    entries = parse_sources_stream(f)
+    f.close()
+    return entries
+
+def dump_sources(path, entries):
+    f = open(path, "w")
+    for name in sorted(entries.keys()):
+        #FIXME Unicode!
+        sum = entries[name]
+        f.write(sum + " " + name + "\n")
+    f.close()
+
+def sources_path(path):
+    # returns the 'sources' file path for a give file path or directory
+    sname = config.get("binrepo", "sources-file", "sources")
+    sdir = path
+    if not os.path.isdir(path):
+        sdir = os.path.dirname(path)
+    spath = os.path.join(sdir, "sources")
+    return spath
+
+def get_chksum(path):
+    sha1 = sha.new()
+    f = open(path)
+    while True:
+        data = f.read(4096)
+        if not data:
+            break
+        sha1.update(data)
+    f.close()
+    digest = sha1.hexdigest()
+    return digest
+
+def update_sources(paths):
+    spath = sources_path(paths[0])
+    entries = parse_sources(spath)
+    added = []
+    deleted = []
+    for path in paths:
+        name = os.path.basename(path)
+        if os.path.exists(path):
+            entries[name] = get_chksum(path)
+            added.append(name)
+        else:
+            deleted.append(name)
+            entries.pop(name, None)
+    dump_sources(spath, entries)
+    return added, deleted
+
+def is_binary(path):
+    raw = config.get("binrepo", "upload-match",
+            "\.(gz|bz2|zip|Z|tar|xar|rpm|7z|lzma)$")
+    maxsize = config.getint("binrepo", "upload-match-size", "1048576")
+    expr = re.compile(raw)
+    name = os.path.basename(path)
+    if expr.search(name):
+        return True
+    st = os.stat(path)
+    if st[stat.ST_SIZE] >= maxsize: # 1MiB
+        return True
+    return False
+
+def find_binaries(paths):
+    new = []
+    for path in paths:
+        if os.path.isdir(path):
+            for name in os.listdir(path):
+                fpath = os.path.join(path, name)
+                if is_binary(fpath):
+                    new.append(fpath)
+        else:
+            if is_binary(path):
+                new.append(path)
+    return new
+
+def is_tracked(path):
+    spath = sources_path(path)
+    entries = parse_sources(spath)
+    name = os.path.basename(path)
+    return name in entries
+
+def upload(paths, auto=False):
+    base = config.get("binrepo", "upload-command", 
+            "/usr/share/repsys/binrepo-upload")
+    if auto:
+        paths = find_binaries(paths)
+    else:
+        for path in paths:
+            if os.path.isdir(path):
+                raise Error, "can't upload directories, try with -a"
+    if not paths:
+        raise Error, "nothing to upload" # is it an error?
+    target = target_url(paths[0])
+    copy(sources=paths, dest=target, makedirs=True)
+    ad = update_sources(paths)
+    return ad
+
+def _binary_list_path(dirpath, name):
+    fpath = os.path.join(dirpath, ".svn", name)
+    return fpath
+
+def _append_binary_list(path, listname):
+    if os.path.isdir(path):
+        raise Error, "only files can be uploaded"
+    basedir = os.path.dirname(path)
+    ppath = _binary_list_path(basedir, listname)
+    f = open(ppath, "a+")
+    f.write(os.path.basename(path) + "\n")
+    f.close()
+
+def _get_binary_list(dirpath, listname):
+    ppath = _binary_list_path(dirpath, listname)
+    if not os.path.exists(ppath):
+        return []
+    f = open(ppath)
+    entries = [line.rstrip() for line in f]
+    f.close()
+    return entries
+
+def _delete_binary_list(dirpath, listname):
+    ppath = _binary_list_path(dirpath, listname)
+    if os.path.exists(ppath):
+        os.unlink(ppath)
+
+def upload_pending(path):
+    _append_binary_list(path, BINLIST_PENDING)
+
+def delete_pending(path):
+    _append_binary_list(path, BINLIST_DELETE)
+
+def commit(dirpath):
+    fetch = (lambda listname:
+                [os.path.join(dirpath, name) for name in
+                    _get_binary_list(dirpath, listname)])
+    pending = fetch(BINLIST_PENDING)
+    delete = fetch(BINLIST_DELETE)
+    if pending:
+        upload(pending)
+        _delete_binary_list(dirpath, BINLIST_PENDING)
+    if delete:
+        remove(delete)
+        _delete_binary_list(dirpath, BINLIST_DELETE)
+
+def remove(paths):
+    # we don't care what will happen to the sources file in the tarballs
+    # repository, we just remove the reference to it
+    spath = sources_path(paths[0])
+    entries = parse_sources(spath)
+    for path in paths:
+        if os.path.exists(path):
+            name = os.path.basename(path)
+            if name not in entries:
+                raise Error, "the file %s is not in the sources list" % path
+            try:
+                os.unlink(path)
+            except (OSError, IOError), e:
+                raise Error, "failed to unlink file: %s" % e
+    ad = update_sources(paths)
+    return ad
+
+def remove_from_sources(path):
+    #FIXME merge with remove() and update_sources()
+    spath = sources_path(path)
+    entries = parse_sources(spath)
+    name = os.path.basename(path)
+    try:
+        del entries[name]
+    except KeyError:
+        pass
+    dump_sources(spath, entries)
+
+def markrelease(srcurl, desturl, version, release, revision):
+    svn = SVN()
+    target_root = target_url()
+    source = target_url(srcurl)
+    root = svn_root(srcurl)
+    relpath = desturl[len(root):]
+    target = os.path.normpath(target_root + "/" + relpath)
+    #XXX rsync doesn't support remote paths in both src and dest, so we
+    # assume we can do it only locally
+    # so we strip the hostname:
+    spath = source[source.find(":")+1:]
+    tpath = target[target.find(":")+1:]
+    tmproot = target_root[target_root.find(":")+1:]
+    sname = config.get("binrepo", "sources-file", "sources")
+    sourcesurl = os.path.join(srcurl, sname)
+    try:
+        stream = StringIO(svn.cat(sourcesurl, rev=revision))
+    except Error:
+        # we don't have a sources file, so there is nothing to copy
+        return
+    entries = parse_sources_stream(stream)
+    paths = [os.path.join(spath, name) for name in entries]
+    # we use target_url as tmproot trying to be 'hardlink friendly'
+    tmpdir = tempfile.mkdtemp(prefix="repsys-markrelease-", dir=tmproot)
+    try:
+        # Check if the files we are going to markrelease are the right
+        # ones.
+        # We copy them to a temporary directory in order to be sure it will
+        # not be changed after we have checked it. Note the comment about
+        # being 'hardlink friendly': we are assuming rsync will also be
+        # hardlink friendly and will create another file even to just
+        # change some file in current/. Of course a dangerous assumption.
+        copy(paths, tmpdir, makedirs=True)
+        tmppaths = []
+        for name, sum in entries.iteritems():
+            path = os.path.join(tmpdir, name)
+            try:
+                check_hash(path, sum)
+            except ChecksumError, e:
+                raise Error, "can't create release: %s" % e
+            tmppaths.append(path)
+        copy(tmppaths, tpath, makedirs=True, hardlink=True)
+    finally:
+        shutil.rmtree(tmpdir)
+
+def download(target, url=None, check=True):
+    targeturl = target_url(url or target)
+    spath = sources_path(target)
+    if not os.path.exists(spath):
+        # we don't have external sources
+        return
+    entries = parse_sources(spath)
+    try:
+        host, path = targeturl.split(":", 1)
+    except ValueError:
+        host = None
+        path = targeturl
+    if os.path.isdir(target):
+        paths = [os.path.join(path, name) for name, sum in entries.iteritems()]
+        targetdir = target
+    else:
+        paths = [os.path.join(path, os.path.basename(target))]
+        name = os.path.basename(target)
+        targetdir = os.path.dirname(target)
+        if name not in entries:
+            raise Error, "file not uploaded yet (not found in "\
+                    "sources file): %s" % target
+    copy(sources=paths, sourcehost=host, dest=targetdir)
+    yield "Checking files"
+    if check:
+        for path in paths:
+            name = os.path.basename(path)
+            bpath = os.path.join(targetdir, name)
+            sum = entries[name]
+            check_hash(bpath, sum)
+    yield "Done"
+
diff --git a/RepSys/commands/co.py b/RepSys/commands/co.py
index 830b7e7..f0b916a 100644
--- a/RepSys/commands/co.py
+++ b/RepSys/commands/co.py
@@ -16,6 +16,9 @@ repository.
 Options:
     -r REV  Revision to checkout
     -o      Do not use the mirror (use official server)
+    -S      Do not download sources from the binaries repository
+    -C      Do not check integrity of files fetched from the binary
+            repository
     -h      Show this message
 
 Examples:
@@ -29,6 +32,10 @@ def parse_options():
     parser.add_option("-r", dest="revision")
     parser.add_option("-o", dest="use_mirror", default=True,
             action="store_false")
+    parser.add_option("-S", dest="use_binrepo", default=True,
+            action="store_false")
+    parser.add_option("-C", dest="binrepo_check", default=True,
+            action="store_false")
     opts, args = parser.parse_args()
     if len(args) not in (1, 2):
         raise Error, "invalid arguments"
diff --git a/RepSys/commands/del.py b/RepSys/commands/del.py
new file mode 100644
index 0000000..2c6902e
--- /dev/null
+++ b/RepSys/commands/del.py
@@ -0,0 +1,30 @@
+from RepSys import Error
+from RepSys.command import *
+from RepSys.rpmutil import binrepo_delete
+
+HELP = """\
+Usage: repsys del [OPTIONS] [PATH]
+
+Remove a given file from the binary sources repository.
+
+Changes in the sources file will be left uncommited.
+
+Options:
+    -c      automatically commit the 'sources' file
+    -h      help
+
+"""
+
+def parse_options():
+    parser = OptionParser(help=HELP)
+    parser.add_option("-c", dest="commit", default=False,
+            action="store_true")
+    opts, args = parser.parse_args()
+    if len(args):
+        opts.paths = args
+    else:
+        raise Error, "you need to provide a path"
+    return opts
+
+def main():
+    do_command(parse_options, binrepo_delete)
diff --git a/RepSys/commands/getsrpm.py b/RepSys/commands/getsrpm.py
index f9a63d2..d6ee8eb 100644
--- a/RepSys/commands/getsrpm.py
+++ b/RepSys/commands/getsrpm.py
@@ -30,6 +30,9 @@ Options:
     -l         Use subversion log to build rpm %changelog
     -T FILE    Template to be used to generate the %changelog
     -h         Show this message
+    -S         Do not download sources from the binary repository
+    -C         Do not check integrity of files fetched from the binary
+               repository
     --strict   Check if the given revision contains changes in REPPKGURL
 
 Examples:
@@ -74,6 +77,10 @@ def parse_options():
     parser.add_option("-n", dest="revname", action="store_true")
     parser.add_option("-l", dest="svnlog", action="store_true")
     parser.add_option("-T", dest="template", type="string", default=None)
+    parser.add_option("-S", dest="use_binrepo", default=True,
+            action="store_false")
+    parser.add_option("-C", dest="binrepo_check", default=True,
+            action="store_false")
     parser.add_option("--strict", dest="strict", default=False,
             action="store_true")
     opts, args = parser.parse_args()
diff --git a/RepSys/commands/sync.py b/RepSys/commands/sync.py
index a51db22..fecf08d 100644
--- a/RepSys/commands/sync.py
+++ b/RepSys/commands/sync.py
@@ -11,6 +11,7 @@ from the spec file.
 "No changes are commited."
 
 Options:
+    -c           Commit the changes, as in ci
     --dry-run    Print results without changing the working copy
     --download -d
                  Try to download the source files not found
@@ -24,6 +25,8 @@ def parse_options():
     parser = OptionParser(help=HELP)
     parser.add_option("--dry-run", dest="dryrun", default=False,
             action="store_true")
+    parser.add_option("-c", dest="ci", default=False,
+            action="store_true")
     parser.add_option("-d", "--download", dest="download", default=False,
             action="store_true")
     opts, args = parser.parse_args()
diff --git a/RepSys/commands/up.py b/RepSys/commands/up.py
new file mode 100644
index 0000000..02a1a9f
--- /dev/null
+++ b/RepSys/commands/up.py
@@ -0,0 +1,22 @@
+from RepSys import Error
+from RepSys.command import *
+from RepSys.rpmutil import update
+
+HELP = """\
+Usage: repsys up [PATH]
+
+Update the package working copy and synchronize all binaries.
+
+Options:
+    -h      help
+"""
+
+def parse_options():
+    parser = OptionParser(help=HELP)
+    opts, args = parser.parse_args()
+    if args:
+        opts.target = args[0]
+    return opts
+
+def main():
+    do_command(parse_options, update)
diff --git a/RepSys/commands/upload.py b/RepSys/commands/upload.py
new file mode 100644
index 0000000..49840a6
--- /dev/null
+++ b/RepSys/commands/upload.py
@@ -0,0 +1,39 @@
+from RepSys import Error
+from RepSys.command import *
+from RepSys.rpmutil import upload
+
+HELP = """\
+Usage: repsys upload [OPTIONS] [PATH]
+
+Upload a given file to the binary sources repository.
+
+It will also update the contents of the 'sources' file and left it
+uncommited.
+
+If the path is a directory, all the contents of the directory will be
+uploaded or removed.
+
+Options:
+    -a      find all possible binary sources inside PATH
+    -c      automatically commit the 'sources' file
+    -A      do not 'svn add' the 'sources' file
+    -h      help
+
+"""
+
+def parse_options():
+    parser = OptionParser(help=HELP)
+    parser.add_option("-c", dest="commit", default=False,
+            action="store_true")
+    parser.add_option("-A", dest="addsources", default=True,
+            action="store_false")
+    parser.add_option("-a", dest="auto", default=False, action="store_true")
+    opts, args = parser.parse_args()
+    if len(args):
+        opts.paths = args
+    else:
+        raise Error, "you need to provide a path"
+    return opts
+
+def main():
+    do_command(parse_options, upload)
diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py
index 3bb1b50..46fa392 100644
--- a/RepSys/rpmutil.py
+++ b/RepSys/rpmutil.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 from RepSys import Error, config, RepSysTree
-from RepSys import mirror
+from RepSys import mirror, binrepo
 from RepSys.svn import SVN
 from RepSys.simplerpm import SRPM
 from RepSys.log import specfile_svn2rpm
@@ -64,7 +64,9 @@ def get_srpm(pkgdirurl,
              template = None,
              macros = [],
              verbose = 0,
-             strict = False):
+             strict = False,
+             use_binrepo = False,
+             binrepo_check = True):
     svn = SVN()
     tmpdir = tempfile.mktemp()
     topdir = "--define '_topdir %s'" % tmpdir
@@ -91,6 +93,8 @@ def get_srpm(pkgdirurl,
             raise Error, "the revision %s does not change anything "\
                     "inside %s" % (revision or "HEAD", geturl)
         svn.export(geturl, tmpdir, rev=revision)
+        if use_binrepo:
+            download_binaries(tmpdir, geturl, check=binrepo_check)
         srpmsdir = os.path.join(tmpdir, "SRPMS")
         os.mkdir(srpmsdir)
         specsdir = os.path.join(tmpdir, "SPECS")
@@ -308,6 +312,10 @@ def mark_release(pkgdirurl, version, release, revision):
     releasesurl = "/".join([pkgdirurl, "releases"])
     versionurl = "/".join([releasesurl, version])
     releaseurl = "/".join([versionurl, release])
+    currenturl = os.path.join(pkgdirurl, "current")
+    cursources = os.path.join(currenturl, "SOURCES")
+    relsources = os.path.join(releaseurl, "SOURCES")
+    binrepo.markrelease(cursources, relsources, version, release, revision)
     if svn.ls(releaseurl, noerror=1):
         raise Error, "release already exists"
     svn.mkdir(releasesurl, noerror=1,
@@ -317,7 +325,6 @@ def mark_release(pkgdirurl, version, release, revision):
     pristineurl = os.path.join(pkgdirurl, "pristine")
     svn.remove(pristineurl, noerror=1,
                log="Removing previous pristine/ directory.")
-    currenturl = os.path.join(pkgdirurl, "current")
     svn.copy(currenturl, pristineurl,
              log="Copying release %s-%s to pristine/ directory." %
                  (version, release))
@@ -385,7 +392,8 @@ def check_changed(pkgdirurl, all=0, show=0, verbose=0):
             "nocurrent": nocurrent,
             "nopristine": nopristine}
 
-def checkout(pkgdirurl, path=None, revision=None, use_mirror=True):
+def checkout(pkgdirurl, path=None, revision=None, use_mirror=True,
+        use_binrepo=False, binrepo_check=True):
     o_pkgdirurl = pkgdirurl
     pkgdirurl = default_parent(o_pkgdirurl)
     current = os.path.join(pkgdirurl, "current")
@@ -398,24 +406,34 @@ def checkout(pkgdirurl, path=None, revision=None, use_mirror=True):
         print "checking out from mirror", current
     svn = SVN()
     svn.checkout(current, path, rev=revision, show=1)
-
-def _getpkgtopdir(basedir=None):
+    if use_binrepo:
+        download_binaries(path, check=binrepo_check)
+    
+def getpkgtopdir(basedir=None):
+    #FIXME this implementation doesn't work well with relative path names,
+    # which is something we need in order to have a friendlier output
     if basedir is None:
-        basedir = os.getcwd()
-    cwd = os.getcwd()
-    dirname = os.path.basename(cwd)
-    if dirname == "SPECS" or dirname == "SOURCES":
-        topdir = os.pardir
-    else:
-        topdir = ""
-    return topdir
+        basedir = os.path.curdir
+    while not ispkgtopdir(basedir):
+        if os.path.abspath(basedir) == "/":
+            raise Error, "can't find top package directories SOURCES and SPECS"
+        basedir = os.path.join(basedir, os.path.pardir)
+    if basedir.startswith("./"):
+        basedir = basedir[2:]
+    return basedir
 
-def sync(dryrun=False, download=False):
+def ispkgtopdir(path=None):
+    if path is None:
+        path = os.getcwd()
+    names = os.listdir(path)
+    return (".svn" in names and "SPECS" in names and "SOURCES" in names)
+
+def sync(dryrun=False, ci=False, download=False):
     svn = SVN()
-    topdir = _getpkgtopdir()
+    topdir = getpkgtopdir()
     # run svn info because svn st does not complain when topdir is not an
     # working copy
-    svn.info(topdir or ".")
+    svn.info(topdir)
     specsdir = os.path.join(topdir, "SPECS/")
     sourcesdir = os.path.join(topdir, "SOURCES/")
     for path in (specsdir, sourcesdir):
@@ -434,15 +452,21 @@ def sync(dryrun=False, download=False):
             for name, no, flags in spec.sources())
     sourcesst = dict((os.path.basename(path), (path, st))
             for st, path in svn.status(sourcesdir, noignore=True))
-    toadd = []
+    toadd_br = []
+    toadd_svn = []
+    toremove_svn = []
+    toremove_br = []
     for source, url in sources.iteritems():
         sourcepath = os.path.join(sourcesdir, source)
-        pst = sourcesst.get(source)
-        if pst:
+        if sourcesst.get(source):
             if os.path.isfile(sourcepath):
-                toadd.append(sourcepath)
+                if not binrepo.is_tracked(sourcepath):
+                    if binrepo.is_binary(sourcepath):
+                        toadd_br.append(sourcepath)
+                    else:
+                        toadd_svn.append(sourcepath)
             else:
-                sys.stderr.write("warning: %s not found, skipping\n" % sourcepath)
+                sys.stderr.write("warning: %s not found\n" % sourcepath)
         elif download and not os.path.isfile(sourcepath):
             print "%s not found, downloading from %s" % (sourcepath, url)
             fmt = config.get("global", "download-command",
@@ -455,29 +479,47 @@ def sync(dryrun=False, download=False):
                         "configuration option" % e
             execcmd(cmd, show=True)
             if os.path.isfile(sourcepath):
-                toadd.append(sourcepath)
+                if binrepo.is_binary(sourcepath):
+                    toadd_br.append(sourcepath)
+                else:
+                    toadd_svn.append(sourcepath)
             else:
                 raise Error, "file not found: %s" % sourcepath
     # rm entries not found in sources and still in svn
     found = os.listdir(sourcesdir)
-    toremove = []
     for entry in found:
-        if entry == ".svn":
+        if entry == ".svn" or entry == "sources":
             continue
         status = sourcesst.get(entry)
-        if status is None and entry not in sources:
-            path = os.path.join(sourcesdir, entry)
-            toremove.append(path)
-    for path in toremove:
+        path = os.path.join(sourcesdir, entry)
+        if entry not in sources:
+            if status is None: # file is tracked by svn
+                toremove_svn.append(path)
+            elif binrepo.is_tracked(path):
+                toremove_br.append(path)
+    for path in toremove_svn:
         print "D\t%s" % path
         if not dryrun:
             svn.remove(path, local=True)
-    for path in toadd:
+    for path in toremove_br:
+        print "DB\t%s" % path
+        if not dryrun:
+            binrepo.delete_pending(path)
+    for path in toadd_svn:
         print "A\t%s" % path
         if not dryrun:
             svn.add(path, local=True)
+    for path in toadd_br:
+        print "AB\t%s" % path
+        if not dryrun:
+            binrepo.upload_pending(path)
+    if commit:
+        commit(topdir)
 
 def commit(target=".", message=None, logfile=None):
+    topdir = getpkgtopdir(target)
+    sourcesdir = os.path.join(topdir, "SOURCES")
+    binrepo.commit(sourcesdir) #TODO make it optional
     svn = SVN()
     status = svn.status(target, quiet=True)
     if not status:
@@ -504,9 +546,90 @@ def commit(target=".", message=None, logfile=None):
         print "use \"repsys switch\" in order to switch back to mirror "\
                 "later"
 
+def download_binaries(target, pkgdirurl=None, check=True):
+    refurl = pkgdirurl
+    if refurl is None:
+        refurl = binrepo.svn_root(target)
+    if binrepo.enabled(refurl):
+        sourcesdir = "SOURCES"
+        url = None
+        bintarget = os.path.join(target, sourcesdir)
+        if pkgdirurl:
+            url = os.path.join(pkgdirurl, sourcesdir)
+        for status in binrepo.download(bintarget, url, check):
+            print status
+
+def update(target=None):
+    svn = SVN()
+    svn_target = None
+    br_target = None
+    if target:
+        info = svn.info2(target) 
+        spath = binrepo.sources_path(target)
+        if info is None:
+            # probably something kept in the binary repository
+            if os.path.exists(spath):
+                entries = binrepo.parse_sources(spath)
+                name = os.path.basename(target)
+                if name in entries:
+                    br_target = target
+                    svn_target = spath
+        else:
+            svn_target = target
+            if info["Node Kind"] == "directory":
+                if os.path.exists(spath):
+                    br_target = target
+    else:
+        top = getpkgtopdir()
+        svn_target = top
+        br_target = os.path.join(top, "SOURCES")
+    if not br_target and not svn_target:
+        raise Error, "target not in SVN nor in binaries "\
+                "repository: %s" % target
+    if svn_target:
+        svn.update(svn_target, show=True)
+    if br_target:
+        for status in binrepo.download(br_target):
+            print status
+
+def _sources_log(added, deleted):
+    lines = ["SILENT: changed sources list:\n"]
+    for name in added:
+        lines.append("A\t" + name)
+    for name in deleted:
+        lines.append("D\t" + name)
+    log = "\n".join(lines)
+    return log
+
+def upload(paths, auto=False, commit=False, addsources=False):
+    if auto and not paths:
+        topdir = getpkgtopdir()
+        paths = [os.path.join(topdir, "SOURCES")]
+    added, deleted = binrepo.upload(paths, auto)
+    if addsources or commit:
+        svn = SVN()
+        spath = binrepo.sources_path(paths[0])
+        if addsources:
+            svn.add(spath)
+        if commit:
+            log = _sources_log(added, deleted)
+            svn.commit(spath, log=log)
+
+def binrepo_delete(paths, commit=False):
+    #TODO handle files tracked by svn
+    refurl = binrepo.svn_root(paths[0])
+    if not binrepo.enabled(refurl):
+        raise Error, "binary repository is not enabled for %s" % refurl
+    added, deleted = binrepo.remove(paths)
+    if commit:
+        svn = SVN()
+        spath = binrepo.sources_path(paths[0])
+        log = _sources_log(added, deleted)
+        svn.commit(spath, log=log)
+
 def switch(mirrorurl=None):
     svn  = SVN()
-    topdir = _getpkgtopdir()
+    topdir = getpkgtopdir()
     info = svn.info2(topdir)
     wcurl = info.get("URL")
     if wcurl is None:
diff --git a/repsys b/repsys
index 67bb180..964750e 100755
--- a/repsys
+++ b/repsys
@@ -4,7 +4,7 @@ from RepSys.command import *
 import getopt
 import sys
 
-VERSION="1.6.20"
+VERSION="1.6.20.8-binrepo"
 
 HELP = """\
 Usage: repsys COMMAND [COMMAND ARGUMENTS]
diff --git a/repsys.conf b/repsys.conf
index 0d1ddc5..624c2ca 100644
--- a/repsys.conf
+++ b/repsys.conf
@@ -2,6 +2,8 @@
 default_parent = svn+ssh://svn.mandriva.com/svn/packages/cooker
 ## uncomment it in case you don't have a account in the Mandriva cluster:
 #mirror = http://svn.mandriva.com/svn/packages/cooker/
+#use-binaries-repository = yes
+#binaries-repository = svn.mandriva.com:/tarballs/
 
 [log]
 oldurl = svn+ssh://svn.mandriva.com/svn/packages/misc