diff options
Diffstat (limited to 'MgaRepo/binrepo.py')
| -rw-r--r-- | MgaRepo/binrepo.py | 395 |
1 files changed, 395 insertions, 0 deletions
diff --git a/MgaRepo/binrepo.py b/MgaRepo/binrepo.py new file mode 100644 index 0000000..9186980 --- /dev/null +++ b/MgaRepo/binrepo.py @@ -0,0 +1,395 @@ +from MgaRepo import Error, config, mirror, layout +from MgaRepo.util import execcmd, rellink +from MgaRepo.svn import SVN + +import sys +import os +import string +import stat +import shutil +import re +import tempfile +import hashlib +import urlparse +import threading +from cStringIO import StringIO + +DEFAULT_TARBALLS_REPO = "/tarballs" +BINARIES_DIR_NAME = "SOURCES" +BINARIES_CHECKOUT_NAME = "SOURCES-bin" + +PROP_USES_BINREPO = "mdv:uses-binrepo" +PROP_BINREPO_REV = "mdv:binrepo-rev" + +BINREPOS_SECTION = "binrepos" + +SOURCES_FILE = "sha1.lst" + +class ChecksumError(Error): + pass + +def svn_baseurl(target): + svn = SVN() + info = svn.info2(target) + if info is None: + # unversioned resource + newtarget = os.path.dirname(target) + info = svn.info2(newtarget) + assert info is not None, "svn_basedir should not be used with a "\ + "non-versioned directory" + root = info["Repository Root"] + url = info["URL"] + kind = info["Node Kind"] + path = url[len(root):] + if kind == "directory": + return url + basepath = os.path.dirname(path) + baseurl = mirror.normalize_path(url + "/" + basepath) + return baseurl + +def svn_root(target): + svn = SVN() + info = svn.info2(target) + if info is None: + newtarget = os.path.dirname(target) + info = svn.info2(newtarget) + assert info is not None + return info["Repository Root"] + +def enabled(url): + #TODO use information from url to find out whether we have a binrepo + # available for this url + use = config.getbool("global", "use-binaries-repository", False) + return use + +def default_repo(): + base = config.get("global", "binaries-repository", None) + if base is None: + default_parent = config.get("global", "default_parent", None) + if default_parent is None: + raise Error, "no binaries-repository nor default_parent "\ + "configured" + comps = urlparse.urlparse(default_parent) + base = comps[1] + ":" + DEFAULT_TARBALLS_REPO + return base + +def translate_url(url): + url = mirror.normalize_path(url) + main = mirror.normalize_path(layout.repository_url()) + subpath = url[len(main)+1:] + # [binrepos] + # updates/2009.0 = svn+ssh://svn.mandriva.com/svn/binrepo/20090/ + ## svn+ssh://svn.mandriva.com/svn/packages/2009.0/trafshow/current + ## would translate to + ## svn+ssh://svn.mandriva.com/svn/binrepo/20090/updates/trafshow/current/ + binbase = None + if BINREPOS_SECTION in config.sections(): + for option, value in config.walk(BINREPOS_SECTION): + if subpath.startswith(option): + binbase = value + break + binurl = mirror._joinurl(binbase or default_repo(), subpath) + return binurl + +def translate_topdir(path): + """Returns the URL in the binrepo from a given path inside a SVN + checkout directory. + + @path: if specified, returns a URL in the binrepo whose path is the + same as the path inside the main repository. + """ + baseurl = svn_baseurl(path) + binurl = translate_url(baseurl) + target = mirror.normalize_path(binurl) + return target + +def is_binary(path): + raw = config.get("binrepo", "upload-match", + "\.(7z|Z|bin|bz2|cpio|db|deb|egg|gem|gz|jar|jisp|lzma|"\ + "pdf|pgn\\.gz|pk3|png|rpm|run|sdz|smzip|tar|tbz|"\ + "tbz2|tgz|ttf|uqm|wad|war|xar|xpi|xz|zip|wav|mp3|ogg|"\ + "jpg|png|gif|avi|mpg|mpeg|rar)$") + maxsize = config.getint("binrepo", "upload-match-size", "1048576") # 1MiB + expr = re.compile(raw) + name = os.path.basename(path) + if expr.search(name): + return True + st = os.stat(path) + if st[stat.ST_SIZE] >= maxsize: + return True + return False + +def find_binaries(paths): + new = [] + for path in paths: + if os.path.isdir(path): + for name in os.listdir(path): + fpath = os.path.join(path, name) + if is_binary(fpath): + new.append(fpath) + else: + if is_binary(path): + new.append(path) + return new + +def make_symlinks(source, dest): + todo = [] + tomove = [] + for name in os.listdir(source): + path = os.path.join(source, name) + if not os.path.isdir(path) and not name.startswith("."): + destpath = os.path.join(dest, name) + linkpath = rellink(path, destpath) + if os.path.exists(destpath): + if (os.path.islink(destpath) and + os.readlink(destpath) == linkpath): + continue + movepath = destpath + ".repsys-moved" + if os.path.exists(movepath): + raise Error, "cannot create symlink, %s already "\ + "exists (%s too)" % (destpath, movepath) + tomove.append((destpath, movepath)) + todo.append((destpath, linkpath)) + for destpath, movepath in tomove: + os.rename(destpath, movepath) + for destpath, linkpath in todo: + os.symlink(linkpath, destpath) + +def download(targetdir, pkgdirurl=None, export=False, show=True, + revision=None, symlinks=True, check=False): + assert not export or (export and pkgdirurl) + svn = SVN() + sourcespath = os.path.join(targetdir, "SOURCES") + binpath = os.path.join(targetdir, BINARIES_CHECKOUT_NAME) + if pkgdirurl: + topurl = translate_url(pkgdirurl) + else: + topurl = translate_topdir(targetdir) + binrev = None + if revision: + if pkgdirurl: + binrev = mapped_revision(pkgdirurl, revision) + else: + binrev = mapped_revision(targetdir, revision, wc=True) + binurl = mirror._joinurl(topurl, BINARIES_DIR_NAME) + if export: + svn.export(binurl, binpath, rev=binrev, show=show) + else: + svn.checkout(binurl, binpath, rev=binrev, show=show) + if symlinks: + make_symlinks(binpath, sourcespath) + if check: + check_sources(targetdir) + +def import_binaries(topdir, pkgname): + """Import all binaries from a given package checkout + + (with pending svn adds) + + @topdir: the path to the svn checkout + """ + svn = SVN() + topurl = translate_topdir(topdir) + sourcesdir = os.path.join(topdir, "SOURCES") + bintopdir = tempfile.mktemp("repsys") + try: + svn.checkout(topurl, bintopdir) + checkout = True + except Error: + bintopdir = tempfile.mkdtemp("repsys") + checkout = False + try: + bindir = os.path.join(bintopdir, BINARIES_DIR_NAME) + if not os.path.exists(bindir): + if checkout: + svn.mkdir(bindir) + else: + os.mkdir(bindir) + binaries = find_binaries([sourcesdir]) + update = update_sources_threaded(topdir, added=binaries) + for path in binaries: + name = os.path.basename(path) + binpath = os.path.join(bindir, name) + os.rename(path, binpath) + try: + svn.remove(path) + except Error: + # file not tracked + svn.revert(path) + if checkout: + svn.add(binpath) + log = "imported binaries for %s" % pkgname + if checkout: + rev = svn.commit(bindir, log=log) + else: + rev = svn.import_(bintopdir, topurl, log=log) + svn.propset(PROP_USES_BINREPO, "yes", topdir) + svn.propset(PROP_BINREPO_REV, str(rev), topdir) + update.join() + svn.add(sources_path(topdir)) + finally: + shutil.rmtree(bintopdir) + +def create_package_dirs(bintopdir): + svn = SVN() + binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME) + silent = config.get("log", "ignore-string", "SILENT") + message = "%s: created binrepo package structure" % silent + svn.mkdir(binurl, log=message, parents=True) + +def parse_sources(path): + entries = {} + f = open(path) + for rawline in f: + line = rawline.strip() + try: + sum, name = line.split(None, 1) + except ValueError: + # failed to unpack, line format error + raise Error, "invalid line in sources file: %s" % rawline + entries[name] = sum + return entries + +def check_hash(path, sum): + newsum = file_hash(path) + if newsum != sum: + raise ChecksumError, "different checksums for %s: expected %s, "\ + "but %s was found" % (path, sum, newsum) + +def check_sources(topdir): + spath = sources_path(topdir) + if not os.path.exists(spath): + raise Error, "'%s' was not found" % spath + entries = parse_sources(spath) + for name, sum in entries.iteritems(): + fpath = os.path.join(topdir, "SOURCES", name) + check_hash(fpath, sum) + +def file_hash(path): + sum = hashlib.sha1() + f = open(path) + while True: + block = f.read(4096) + if not block: + break + sum.update(block) + f.close() + return sum.hexdigest() + +def sources_path(topdir): + path = os.path.join(topdir, "SOURCES", SOURCES_FILE) + return path + +def update_sources(topdir, added=[], removed=[]): + path = sources_path(topdir) + entries = {} + if os.path.isfile(path): + entries = parse_sources(path) + f = open(path, "w") # open before calculating hashes + for name in removed: + entries.pop(removed) + for added_path in added: + name = os.path.basename(added_path) + entries[name] = file_hash(added_path) + for name in sorted(entries): + f.write("%s %s\n" % (entries[name], name)) + f.close() + +def update_sources_threaded(*args, **kwargs): + t = threading.Thread(target=update_sources, args=args, kwargs=kwargs) + t.start() + t.join() + return t + +def upload(path, message=None): + from MgaRepo.rpmutil import getpkgtopdir + svn = SVN() + if not os.path.exists(path): + raise Error, "not found: %s" % path + # XXX check if the path is under SOURCES/ + paths = find_binaries([path]) + if not paths: + raise Error, "'%s' does not seem to have any tarballs" % path + topdir = getpkgtopdir() + bintopdir = translate_topdir(topdir) + binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME) + sourcesdir = os.path.join(topdir, "SOURCES") + bindir = os.path.join(topdir, BINARIES_CHECKOUT_NAME) + silent = config.get("log", "ignore-string", "SILENT") + if not os.path.exists(bindir): + try: + download(topdir, show=False) + except Error: + # possibly the package does not exist + # (TODO check whether it is really a 'path not found' error) + pass + if not os.path.exists(bindir): + create_package_dirs(bintopdir) + svn.propset(PROP_USES_BINREPO, "yes", topdir) + svn.commit(topdir, log="%s: created binrepo structure" % silent) + download(topdir, show=False) + for path in paths: + if svn.info2(path): + sys.stderr.write("'%s' is already tracked by svn, ignoring\n" % + path) + continue + name = os.path.basename(path) + binpath = os.path.join(bindir, name) + os.rename(path, binpath) + svn.add(binpath) + if not message: + message = "%s: new binary files %s" % (silent, " ".join(paths)) + make_symlinks(bindir, sourcesdir) + update = update_sources_threaded(topdir, added=paths) + rev = svn.commit(binpath, log=message) + svn.propset(PROP_BINREPO_REV, str(rev), topdir) + sources = sources_path(topdir) + svn.add(sources) + update.join() + svn.commit(topdir + " " + sources, log=message, nonrecursive=True) + +def mapped_revision(target, revision, wc=False): + """Maps a txtrepo revision to a binrepo datespec + + This datespec can is intended to be used by svn .. -r DATE. + + @target: a working copy path or a URL + @revision: if target is a URL, the revision number used when fetching + svn info + @wc: if True indicates that 'target' must be interpreted as a + the path of a svn working copy, otherwise it is handled as a URL + """ + svn = SVN() + binrev = None + if wc: + spath = sources_path(target) + if os.path.exists(spath): + infolines = svn.info(spath, xml=True) + if infolines: + rawinfo = "".join(infolines) # arg! + found = re.search("<date>(.*?)</date>", rawinfo).groups() + date = found[0] + else: + raise Error, "bogus 'svn info' for '%s'" % spath + else: + raise Error, "'%s' was not found" % spath + else: + url = mirror._joinurl(target, sources_path("")) + date = svn.propget("svn:date", url, rev=revision, revprop=True) + if not date: + raise Error, "no valid date available for '%s'" % url + binrev = "{%s}" % date + return binrev + +def markrelease(sourceurl, releasesurl, version, release, revision): + svn = SVN() + binrev = mapped_revision(sourceurl, revision) + binsource = translate_url(sourceurl) + binreleases = translate_url(releasesurl) + versiondir = mirror._joinurl(binreleases, version) + dest = mirror._joinurl(versiondir, release) + svn.mkdir(binreleases, noerror=1, log="created directory for releases") + svn.mkdir(versiondir, noerror=1, log="created directory for version %s" % version) + svn.copy(binsource, dest, rev=binrev, + log="%%markrelease ver=%s rel=%s rev=%s binrev=%s" % (version, release, + revision, binrev)) |