aboutsummaryrefslogtreecommitdiffstats
path: root/MgaRepo/binrepo.py
diff options
context:
space:
mode:
Diffstat (limited to 'MgaRepo/binrepo.py')
-rw-r--r--MgaRepo/binrepo.py395
1 files changed, 395 insertions, 0 deletions
diff --git a/MgaRepo/binrepo.py b/MgaRepo/binrepo.py
new file mode 100644
index 0000000..9186980
--- /dev/null
+++ b/MgaRepo/binrepo.py
@@ -0,0 +1,395 @@
+from MgaRepo import Error, config, mirror, layout
+from MgaRepo.util import execcmd, rellink
+from MgaRepo.svn import SVN
+
+import sys
+import os
+import string
+import stat
+import shutil
+import re
+import tempfile
+import hashlib
+import urlparse
+import threading
+from cStringIO import StringIO
+
+DEFAULT_TARBALLS_REPO = "/tarballs"
+BINARIES_DIR_NAME = "SOURCES"
+BINARIES_CHECKOUT_NAME = "SOURCES-bin"
+
+PROP_USES_BINREPO = "mdv:uses-binrepo"
+PROP_BINREPO_REV = "mdv:binrepo-rev"
+
+BINREPOS_SECTION = "binrepos"
+
+SOURCES_FILE = "sha1.lst"
+
+class ChecksumError(Error):
+ pass
+
+def svn_baseurl(target):
+ svn = SVN()
+ info = svn.info2(target)
+ if info is None:
+ # unversioned resource
+ newtarget = os.path.dirname(target)
+ info = svn.info2(newtarget)
+ assert info is not None, "svn_basedir should not be used with a "\
+ "non-versioned directory"
+ root = info["Repository Root"]
+ url = info["URL"]
+ kind = info["Node Kind"]
+ path = url[len(root):]
+ if kind == "directory":
+ return url
+ basepath = os.path.dirname(path)
+ baseurl = mirror.normalize_path(url + "/" + basepath)
+ return baseurl
+
+def svn_root(target):
+ svn = SVN()
+ info = svn.info2(target)
+ if info is None:
+ newtarget = os.path.dirname(target)
+ info = svn.info2(newtarget)
+ assert info is not None
+ return info["Repository Root"]
+
+def enabled(url):
+ #TODO use information from url to find out whether we have a binrepo
+ # available for this url
+ use = config.getbool("global", "use-binaries-repository", False)
+ return use
+
+def default_repo():
+ base = config.get("global", "binaries-repository", None)
+ if base is None:
+ default_parent = config.get("global", "default_parent", None)
+ if default_parent is None:
+ raise Error, "no binaries-repository nor default_parent "\
+ "configured"
+ comps = urlparse.urlparse(default_parent)
+ base = comps[1] + ":" + DEFAULT_TARBALLS_REPO
+ return base
+
+def translate_url(url):
+ url = mirror.normalize_path(url)
+ main = mirror.normalize_path(layout.repository_url())
+ subpath = url[len(main)+1:]
+ # [binrepos]
+ # updates/2009.0 = svn+ssh://svn.mandriva.com/svn/binrepo/20090/
+ ## svn+ssh://svn.mandriva.com/svn/packages/2009.0/trafshow/current
+ ## would translate to
+ ## svn+ssh://svn.mandriva.com/svn/binrepo/20090/updates/trafshow/current/
+ binbase = None
+ if BINREPOS_SECTION in config.sections():
+ for option, value in config.walk(BINREPOS_SECTION):
+ if subpath.startswith(option):
+ binbase = value
+ break
+ binurl = mirror._joinurl(binbase or default_repo(), subpath)
+ return binurl
+
+def translate_topdir(path):
+ """Returns the URL in the binrepo from a given path inside a SVN
+ checkout directory.
+
+ @path: if specified, returns a URL in the binrepo whose path is the
+ same as the path inside the main repository.
+ """
+ baseurl = svn_baseurl(path)
+ binurl = translate_url(baseurl)
+ target = mirror.normalize_path(binurl)
+ return target
+
+def is_binary(path):
+ raw = config.get("binrepo", "upload-match",
+ "\.(7z|Z|bin|bz2|cpio|db|deb|egg|gem|gz|jar|jisp|lzma|"\
+ "pdf|pgn\\.gz|pk3|png|rpm|run|sdz|smzip|tar|tbz|"\
+ "tbz2|tgz|ttf|uqm|wad|war|xar|xpi|xz|zip|wav|mp3|ogg|"\
+ "jpg|png|gif|avi|mpg|mpeg|rar)$")
+ maxsize = config.getint("binrepo", "upload-match-size", "1048576") # 1MiB
+ expr = re.compile(raw)
+ name = os.path.basename(path)
+ if expr.search(name):
+ return True
+ st = os.stat(path)
+ if st[stat.ST_SIZE] >= maxsize:
+ return True
+ return False
+
+def find_binaries(paths):
+ new = []
+ for path in paths:
+ if os.path.isdir(path):
+ for name in os.listdir(path):
+ fpath = os.path.join(path, name)
+ if is_binary(fpath):
+ new.append(fpath)
+ else:
+ if is_binary(path):
+ new.append(path)
+ return new
+
+def make_symlinks(source, dest):
+ todo = []
+ tomove = []
+ for name in os.listdir(source):
+ path = os.path.join(source, name)
+ if not os.path.isdir(path) and not name.startswith("."):
+ destpath = os.path.join(dest, name)
+ linkpath = rellink(path, destpath)
+ if os.path.exists(destpath):
+ if (os.path.islink(destpath) and
+ os.readlink(destpath) == linkpath):
+ continue
+ movepath = destpath + ".repsys-moved"
+ if os.path.exists(movepath):
+ raise Error, "cannot create symlink, %s already "\
+ "exists (%s too)" % (destpath, movepath)
+ tomove.append((destpath, movepath))
+ todo.append((destpath, linkpath))
+ for destpath, movepath in tomove:
+ os.rename(destpath, movepath)
+ for destpath, linkpath in todo:
+ os.symlink(linkpath, destpath)
+
+def download(targetdir, pkgdirurl=None, export=False, show=True,
+ revision=None, symlinks=True, check=False):
+ assert not export or (export and pkgdirurl)
+ svn = SVN()
+ sourcespath = os.path.join(targetdir, "SOURCES")
+ binpath = os.path.join(targetdir, BINARIES_CHECKOUT_NAME)
+ if pkgdirurl:
+ topurl = translate_url(pkgdirurl)
+ else:
+ topurl = translate_topdir(targetdir)
+ binrev = None
+ if revision:
+ if pkgdirurl:
+ binrev = mapped_revision(pkgdirurl, revision)
+ else:
+ binrev = mapped_revision(targetdir, revision, wc=True)
+ binurl = mirror._joinurl(topurl, BINARIES_DIR_NAME)
+ if export:
+ svn.export(binurl, binpath, rev=binrev, show=show)
+ else:
+ svn.checkout(binurl, binpath, rev=binrev, show=show)
+ if symlinks:
+ make_symlinks(binpath, sourcespath)
+ if check:
+ check_sources(targetdir)
+
+def import_binaries(topdir, pkgname):
+ """Import all binaries from a given package checkout
+
+ (with pending svn adds)
+
+ @topdir: the path to the svn checkout
+ """
+ svn = SVN()
+ topurl = translate_topdir(topdir)
+ sourcesdir = os.path.join(topdir, "SOURCES")
+ bintopdir = tempfile.mktemp("repsys")
+ try:
+ svn.checkout(topurl, bintopdir)
+ checkout = True
+ except Error:
+ bintopdir = tempfile.mkdtemp("repsys")
+ checkout = False
+ try:
+ bindir = os.path.join(bintopdir, BINARIES_DIR_NAME)
+ if not os.path.exists(bindir):
+ if checkout:
+ svn.mkdir(bindir)
+ else:
+ os.mkdir(bindir)
+ binaries = find_binaries([sourcesdir])
+ update = update_sources_threaded(topdir, added=binaries)
+ for path in binaries:
+ name = os.path.basename(path)
+ binpath = os.path.join(bindir, name)
+ os.rename(path, binpath)
+ try:
+ svn.remove(path)
+ except Error:
+ # file not tracked
+ svn.revert(path)
+ if checkout:
+ svn.add(binpath)
+ log = "imported binaries for %s" % pkgname
+ if checkout:
+ rev = svn.commit(bindir, log=log)
+ else:
+ rev = svn.import_(bintopdir, topurl, log=log)
+ svn.propset(PROP_USES_BINREPO, "yes", topdir)
+ svn.propset(PROP_BINREPO_REV, str(rev), topdir)
+ update.join()
+ svn.add(sources_path(topdir))
+ finally:
+ shutil.rmtree(bintopdir)
+
+def create_package_dirs(bintopdir):
+ svn = SVN()
+ binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME)
+ silent = config.get("log", "ignore-string", "SILENT")
+ message = "%s: created binrepo package structure" % silent
+ svn.mkdir(binurl, log=message, parents=True)
+
+def parse_sources(path):
+ entries = {}
+ f = open(path)
+ for rawline in f:
+ line = rawline.strip()
+ try:
+ sum, name = line.split(None, 1)
+ except ValueError:
+ # failed to unpack, line format error
+ raise Error, "invalid line in sources file: %s" % rawline
+ entries[name] = sum
+ return entries
+
+def check_hash(path, sum):
+ newsum = file_hash(path)
+ if newsum != sum:
+ raise ChecksumError, "different checksums for %s: expected %s, "\
+ "but %s was found" % (path, sum, newsum)
+
+def check_sources(topdir):
+ spath = sources_path(topdir)
+ if not os.path.exists(spath):
+ raise Error, "'%s' was not found" % spath
+ entries = parse_sources(spath)
+ for name, sum in entries.iteritems():
+ fpath = os.path.join(topdir, "SOURCES", name)
+ check_hash(fpath, sum)
+
+def file_hash(path):
+ sum = hashlib.sha1()
+ f = open(path)
+ while True:
+ block = f.read(4096)
+ if not block:
+ break
+ sum.update(block)
+ f.close()
+ return sum.hexdigest()
+
+def sources_path(topdir):
+ path = os.path.join(topdir, "SOURCES", SOURCES_FILE)
+ return path
+
+def update_sources(topdir, added=[], removed=[]):
+ path = sources_path(topdir)
+ entries = {}
+ if os.path.isfile(path):
+ entries = parse_sources(path)
+ f = open(path, "w") # open before calculating hashes
+ for name in removed:
+ entries.pop(removed)
+ for added_path in added:
+ name = os.path.basename(added_path)
+ entries[name] = file_hash(added_path)
+ for name in sorted(entries):
+ f.write("%s %s\n" % (entries[name], name))
+ f.close()
+
+def update_sources_threaded(*args, **kwargs):
+ t = threading.Thread(target=update_sources, args=args, kwargs=kwargs)
+ t.start()
+ t.join()
+ return t
+
+def upload(path, message=None):
+ from MgaRepo.rpmutil import getpkgtopdir
+ svn = SVN()
+ if not os.path.exists(path):
+ raise Error, "not found: %s" % path
+ # XXX check if the path is under SOURCES/
+ paths = find_binaries([path])
+ if not paths:
+ raise Error, "'%s' does not seem to have any tarballs" % path
+ topdir = getpkgtopdir()
+ bintopdir = translate_topdir(topdir)
+ binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME)
+ sourcesdir = os.path.join(topdir, "SOURCES")
+ bindir = os.path.join(topdir, BINARIES_CHECKOUT_NAME)
+ silent = config.get("log", "ignore-string", "SILENT")
+ if not os.path.exists(bindir):
+ try:
+ download(topdir, show=False)
+ except Error:
+ # possibly the package does not exist
+ # (TODO check whether it is really a 'path not found' error)
+ pass
+ if not os.path.exists(bindir):
+ create_package_dirs(bintopdir)
+ svn.propset(PROP_USES_BINREPO, "yes", topdir)
+ svn.commit(topdir, log="%s: created binrepo structure" % silent)
+ download(topdir, show=False)
+ for path in paths:
+ if svn.info2(path):
+ sys.stderr.write("'%s' is already tracked by svn, ignoring\n" %
+ path)
+ continue
+ name = os.path.basename(path)
+ binpath = os.path.join(bindir, name)
+ os.rename(path, binpath)
+ svn.add(binpath)
+ if not message:
+ message = "%s: new binary files %s" % (silent, " ".join(paths))
+ make_symlinks(bindir, sourcesdir)
+ update = update_sources_threaded(topdir, added=paths)
+ rev = svn.commit(binpath, log=message)
+ svn.propset(PROP_BINREPO_REV, str(rev), topdir)
+ sources = sources_path(topdir)
+ svn.add(sources)
+ update.join()
+ svn.commit(topdir + " " + sources, log=message, nonrecursive=True)
+
+def mapped_revision(target, revision, wc=False):
+ """Maps a txtrepo revision to a binrepo datespec
+
+ This datespec can is intended to be used by svn .. -r DATE.
+
+ @target: a working copy path or a URL
+ @revision: if target is a URL, the revision number used when fetching
+ svn info
+ @wc: if True indicates that 'target' must be interpreted as a
+ the path of a svn working copy, otherwise it is handled as a URL
+ """
+ svn = SVN()
+ binrev = None
+ if wc:
+ spath = sources_path(target)
+ if os.path.exists(spath):
+ infolines = svn.info(spath, xml=True)
+ if infolines:
+ rawinfo = "".join(infolines) # arg!
+ found = re.search("<date>(.*?)</date>", rawinfo).groups()
+ date = found[0]
+ else:
+ raise Error, "bogus 'svn info' for '%s'" % spath
+ else:
+ raise Error, "'%s' was not found" % spath
+ else:
+ url = mirror._joinurl(target, sources_path(""))
+ date = svn.propget("svn:date", url, rev=revision, revprop=True)
+ if not date:
+ raise Error, "no valid date available for '%s'" % url
+ binrev = "{%s}" % date
+ return binrev
+
+def markrelease(sourceurl, releasesurl, version, release, revision):
+ svn = SVN()
+ binrev = mapped_revision(sourceurl, revision)
+ binsource = translate_url(sourceurl)
+ binreleases = translate_url(releasesurl)
+ versiondir = mirror._joinurl(binreleases, version)
+ dest = mirror._joinurl(versiondir, release)
+ svn.mkdir(binreleases, noerror=1, log="created directory for releases")
+ svn.mkdir(versiondir, noerror=1, log="created directory for version %s" % version)
+ svn.copy(binsource, dest, rev=binrev,
+ log="%%markrelease ver=%s rel=%s rev=%s binrev=%s" % (version, release,
+ revision, binrev))