Initial work on the support of the binary files repository

It introduces a command "upload" to explicitly upload tarballs to the
repository and update the list on svn (the file SOURCES/sources). Also a
"del" command, to remove the reference to the file in the sources file. An
another "up", to pull possibly new tarballs from the binaries repository.

The command sync will automatically mark the new files that should go the
binaries repository. They should be uploaded with ci.

With the current code, packages can have tarballs gradually migrated as
tarballs of newer versions will appear.

In this implementation markrelease is required to be run in the same
machine of the repository (or at least should have direct fs access to it),
as it works by performing plain "cp"

1 files changed, 425 insertions, 0 deletions

diff --git a/RepSys/binrepo.py b/RepSys/binrepo.py
new file mode 100644
index 0000000..9fbd19e
--- /dev/null
+++ b/RepSys/binrepo.py
@@ -0,0 +1,425 @@
+from RepSys import Error, RepSysTree, config
+from RepSys.util import execcmd
+from RepSys.svn import SVN
+from RepSys.mirror import same_base
+
+import os
+import string
+import stat
+import sha
+import shutil
+import re
+import tempfile
+import urlparse
+from cStringIO import StringIO
+
+#TODO logging for markrelease
+
+DEFAULT_TARBALLS_REPO = "/tarballs"
+BINLIST_PENDING = "repsys-upload"
+BINLIST_DELETE = "repsys-delete"
+
+class ChecksumError(Error):
+    pass
+
+def copy_rsync(sources, dest, sourcehost=None, desthost=None,
+        archive=False, recurse=False):
+    """Simple inteface for rsync"""
+    args = ["rsync", "-i", "--log-format=\"%i %n\""]
+    if archive:
+        args.append("-a")
+    if recurse:
+        args.append("-r")
+    if sourcehost:
+        # "svn.mandriva.com:/foo/a /foo/b"
+        #TODO space escaping needed for sources
+        args.append("\"" + sourcehost + ":" + " ".join(sources) + "\"")
+    else:
+        args.extend(sources)
+    if desthost:
+        args.append(desthost + ":" + dest)
+    else:
+        args.append(dest)
+    execcmd(show=True, *args)
+
+def makedirs_remote(path, host):
+    tmpdir = tempfile.mkdtemp(prefix="repsys-makedirs")
+    try:
+        newpath = os.path.normpath(tmpdir + "/" + path)
+        os.makedirs(newpath)
+        copy_rsync(sources=[tmpdir + "/"], dest="/", desthost=host, recurse=True)
+    finally:
+        if os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
+
+def copy(sources, dest, sourcehost=None, desthost=None, makedirs=False,
+        hardlink=False):
+    """rsync-like copy
+
+    Note that a copy between two dirs will result in overwriting the
+    latter.
+    """
+    if desthost is None:
+        # we only need dest to contain the host name
+        try:
+            desthost, dpath = dest.split(":", 1)
+        except ValueError:
+            dpath = dest
+    if makedirs:
+        if desthost:
+            makedirs_remote(dpath, desthost)
+        else:
+            try:
+                os.makedirs(dpath)
+            except OSError, e:
+                if e.errno != 17: # already exists
+                    raise
+    if sourcehost or desthost:
+        copy_rsync(sources=sources, sourcehost=sourcehost, dest=dpath,
+                desthost=desthost, recurse=True, archive=True)
+    else:
+        opts = "-a"
+        if hardlink:
+            opts += "l"
+        for source in sources:
+            if os.path.isdir(source) and os.path.exists(dpath):
+                #FIXME ugly workaround to behave consistently between
+                # remote and local copies:
+                try:
+                    os.rmdir(dpath)
+                except OSError, e:
+                    raise Error, "can't overwrite directory: %s" % e
+            execcmd("cp %s %s %s" % (opts, source, dpath))
+
+def svn_basedir(target):
+    svn = SVN()
+    info = svn.info2(target)
+    if info is None:
+        # unversioned resource
+        newtarget = os.path.dirname(target)
+        info = svn.info2(newtarget)
+        assert info is not None, "svn_basedir should not be used with a "\
+                "non-versioned directory"
+    root = info["Repository Root"]
+    url = info["URL"]
+    kind = info["Node Kind"]
+    path = url[len(root):]
+    if kind == "directory":
+        return path
+    return os.path.dirname(path)
+
+def svn_root(target):
+    svn = SVN()
+    info = svn.info2(target)
+    if info is None:
+        newtarget = os.path.dirname(target)
+        info = svn.info2(newtarget)
+        assert info is not None
+    return info["Repository Root"]
+
+def enabled(url):
+    use = config.getbool("global", "use-binaries-repository", False)
+    default_parent = config.get("global", "default_parent", None)
+    if url and use and default_parent and same_base(url, default_parent):
+        return True
+    return False
+
+def target_url(path=None):
+    from RepSys.rpmutil import get_submit_info
+    base = config.get("global", "binaries-repository", None)
+    if base is None:
+        default_parent = config.get("global", "default_parent", None)
+        if default_parent is None:
+            raise Error, "no binaries-repository nor default_parent "\
+                    "configured"
+        comps = urlparse.urlparse(default_parent)
+        base = comps[1] + ":" + DEFAULT_TARBALLS_REPO
+    if path:
+        target = os.path.normpath(base + "/" + svn_basedir(path))
+    else:
+        target = base
+    return target
+
+def file_hash(path):
+    sum = sha.new()
+    f = open(path)
+    while True:
+        block = f.read(4096)
+        if not block:
+            break
+        sum.update(block)
+    f.close()
+    return sum.hexdigest()
+
+def check_hash(path, sum):
+    newsum = file_hash(path)
+    if newsum != sum:
+        raise ChecksumError, "different checksums for %s: expected %s, "\
+                "but %s was found" % (path, sum, newsum)
+
+def parse_sources_stream(stream):
+    entries = {}
+    for rawline in stream:
+        line = rawline.strip()
+        try:
+            sum, name = line.split(None, 1)
+        except ValueError:
+            # failed to unpack, line format error
+            raise Error, "invalid line in sources file: %s" % rawline
+        entries[name] = sum
+    return entries
+
+def parse_sources(path, force=False):
+    if not os.path.exists(path) and not force:
+        return {}
+    basedir = os.path.dirname(path)
+    spath = os.path.join(basedir, "sources")
+    f = open(spath)
+    entries = parse_sources_stream(f)
+    f.close()
+    return entries
+
+def dump_sources(path, entries):
+    f = open(path, "w")
+    for name in sorted(entries.keys()):
+        #FIXME Unicode!
+        sum = entries[name]
+        f.write(sum + " " + name + "\n")
+    f.close()
+
+def sources_path(path):
+    # returns the 'sources' file path for a give file path or directory
+    sname = config.get("binrepo", "sources-file", "sources")
+    sdir = path
+    if not os.path.isdir(path):
+        sdir = os.path.dirname(path)
+    spath = os.path.join(sdir, "sources")
+    return spath
+
+def get_chksum(path):
+    sha1 = sha.new()
+    f = open(path)
+    while True:
+        data = f.read(4096)
+        if not data:
+            break
+        sha1.update(data)
+    f.close()
+    digest = sha1.hexdigest()
+    return digest
+
+def update_sources(paths):
+    spath = sources_path(paths[0])
+    entries = parse_sources(spath)
+    added = []
+    deleted = []
+    for path in paths:
+        name = os.path.basename(path)
+        if os.path.exists(path):
+            entries[name] = get_chksum(path)
+            added.append(name)
+        else:
+            deleted.append(name)
+            entries.pop(name, None)
+    dump_sources(spath, entries)
+    return added, deleted
+
+def is_binary(path):
+    raw = config.get("binrepo", "upload-match",
+            "\.(gz|bz2|zip|Z|tar|xar|rpm|7z|lzma)$")
+    maxsize = config.getint("binrepo", "upload-match-size", "1048576")
+    expr = re.compile(raw)
+    name = os.path.basename(path)
+    if expr.search(name):
+        return True
+    st = os.stat(path)
+    if st[stat.ST_SIZE] >= maxsize: # 1MiB
+        return True
+    return False
+
+def find_binaries(paths):
+    new = []
+    for path in paths:
+        if os.path.isdir(path):
+            for name in os.listdir(path):
+                fpath = os.path.join(path, name)
+                if is_binary(fpath):
+                    new.append(fpath)
+        else:
+            if is_binary(path):
+                new.append(path)
+    return new
+
+def is_tracked(path):
+    spath = sources_path(path)
+    entries = parse_sources(spath)
+    name = os.path.basename(path)
+    return name in entries
+
+def upload(paths, auto=False):
+    base = config.get("binrepo", "upload-command", 
+            "/usr/share/repsys/binrepo-upload")
+    if auto:
+        paths = find_binaries(paths)
+    else:
+        for path in paths:
+            if os.path.isdir(path):
+                raise Error, "can't upload directories, try with -a"
+    if not paths:
+        raise Error, "nothing to upload" # is it an error?
+    target = target_url(paths[0])
+    copy(sources=paths, dest=target, makedirs=True)
+    ad = update_sources(paths)
+    return ad
+
+def _binary_list_path(dirpath, name):
+    fpath = os.path.join(dirpath, ".svn", name)
+    return fpath
+
+def _append_binary_list(path, listname):
+    if os.path.isdir(path):
+        raise Error, "only files can be uploaded"
+    basedir = os.path.dirname(path)
+    ppath = _binary_list_path(basedir, listname)
+    f = open(ppath, "a+")
+    f.write(os.path.basename(path) + "\n")
+    f.close()
+
+def _get_binary_list(dirpath, listname):
+    ppath = _binary_list_path(dirpath, listname)
+    if not os.path.exists(ppath):
+        return []
+    f = open(ppath)
+    entries = [line.rstrip() for line in f]
+    f.close()
+    return entries
+
+def _delete_binary_list(dirpath, listname):
+    ppath = _binary_list_path(dirpath, listname)
+    if os.path.exists(ppath):
+        os.unlink(ppath)
+
+def upload_pending(path):
+    _append_binary_list(path, BINLIST_PENDING)
+
+def delete_pending(path):
+    _append_binary_list(path, BINLIST_DELETE)
+
+def commit(dirpath):
+    fetch = (lambda listname:
+                [os.path.join(dirpath, name) for name in
+                    _get_binary_list(dirpath, listname)])
+    pending = fetch(BINLIST_PENDING)
+    delete = fetch(BINLIST_DELETE)
+    if pending:
+        upload(pending)
+        _delete_binary_list(dirpath, BINLIST_PENDING)
+    if delete:
+        remove(delete)
+        _delete_binary_list(dirpath, BINLIST_DELETE)
+
+def remove(paths):
+    # we don't care what will happen to the sources file in the tarballs
+    # repository, we just remove the reference to it
+    spath = sources_path(paths[0])
+    entries = parse_sources(spath)
+    for path in paths:
+        if os.path.exists(path):
+            name = os.path.basename(path)
+            if name not in entries:
+                raise Error, "the file %s is not in the sources list" % path
+            try:
+                os.unlink(path)
+            except (OSError, IOError), e:
+                raise Error, "failed to unlink file: %s" % e
+    ad = update_sources(paths)
+    return ad
+
+def remove_from_sources(path):
+    #FIXME merge with remove() and update_sources()
+    spath = sources_path(path)
+    entries = parse_sources(spath)
+    name = os.path.basename(path)
+    try:
+        del entries[name]
+    except KeyError:
+        pass
+    dump_sources(spath, entries)
+
+def markrelease(srcurl, desturl, version, release, revision):
+    svn = SVN()
+    target_root = target_url()
+    source = target_url(srcurl)
+    root = svn_root(srcurl)
+    relpath = desturl[len(root):]
+    target = os.path.normpath(target_root + "/" + relpath)
+    #XXX rsync doesn't support remote paths in both src and dest, so we
+    # assume we can do it only locally
+    # so we strip the hostname:
+    spath = source[source.find(":")+1:]
+    tpath = target[target.find(":")+1:]
+    tmproot = target_root[target_root.find(":")+1:]
+    sname = config.get("binrepo", "sources-file", "sources")
+    sourcesurl = os.path.join(srcurl, sname)
+    try:
+        stream = StringIO(svn.cat(sourcesurl, rev=revision))
+    except Error:
+        # we don't have a sources file, so there is nothing to copy
+        return
+    entries = parse_sources_stream(stream)
+    paths = [os.path.join(spath, name) for name in entries]
+    # we use target_url as tmproot trying to be 'hardlink friendly'
+    tmpdir = tempfile.mkdtemp(prefix="repsys-markrelease-", dir=tmproot)
+    try:
+        # Check if the files we are going to markrelease are the right
+        # ones.
+        # We copy them to a temporary directory in order to be sure it will
+        # not be changed after we have checked it. Note the comment about
+        # being 'hardlink friendly': we are assuming rsync will also be
+        # hardlink friendly and will create another file even to just
+        # change some file in current/. Of course a dangerous assumption.
+        copy(paths, tmpdir, makedirs=True)
+        tmppaths = []
+        for name, sum in entries.iteritems():
+            path = os.path.join(tmpdir, name)
+            try:
+                check_hash(path, sum)
+            except ChecksumError, e:
+                raise Error, "can't create release: %s" % e
+            tmppaths.append(path)
+        copy(tmppaths, tpath, makedirs=True, hardlink=True)
+    finally:
+        shutil.rmtree(tmpdir)
+
+def download(target, url=None, check=True):
+    targeturl = target_url(url or target)
+    spath = sources_path(target)
+    if not os.path.exists(spath):
+        # we don't have external sources
+        return
+    entries = parse_sources(spath)
+    try:
+        host, path = targeturl.split(":", 1)
+    except ValueError:
+        host = None
+        path = targeturl
+    if os.path.isdir(target):
+        paths = [os.path.join(path, name) for name, sum in entries.iteritems()]
+        targetdir = target
+    else:
+        paths = [os.path.join(path, os.path.basename(target))]
+        name = os.path.basename(target)
+        targetdir = os.path.dirname(target)
+        if name not in entries:
+            raise Error, "file not uploaded yet (not found in "\
+                    "sources file): %s" % target
+    copy(sources=paths, sourcehost=host, dest=targetdir)
+    yield "Checking files"
+    if check:
+        for path in paths:
+            name = os.path.basename(path)
+            bpath = os.path.join(targetdir, name)
+            sum = entries[name]
+            check_hash(bpath, sum)
+    yield "Done"
+