diff options
Diffstat (limited to 'RepSys')
| -rwxr-xr-x[-rw-r--r--] | RepSys/ConfigParser.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/__init__.py | 5 | ||||
| -rwxr-xr-x | RepSys/binrepo.py | 393 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/cgi/__init__.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/cgi/soapserver.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/cgi/submit.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/cgi/xmlrpcserver.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/cgiutil.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/command.py | 4 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/__init__.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/authoremail.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/changed.py | 4 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/ci.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/co.py | 16 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/create.py | 0 | ||||
| -rwxr-xr-x | RepSys/commands/del.py | 30 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/editlog.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/getspec.py | 4 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/getsrpm.py | 10 | ||||
| -rwxr-xr-x | RepSys/commands/log.py | 62 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/markrelease.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/patchspec.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/putsrpm.py | 1 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/rpmlog.py | 4 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/submit.py | 76 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/switch.py | 6 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/commands/sync.py | 9 | ||||
| -rwxr-xr-x | RepSys/commands/up.py | 22 | ||||
| -rwxr-xr-x | RepSys/commands/upload.py | 28 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/layout.py | 2 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/log.py | 20 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/mirror.py | 7 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/plugins/__init__.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/plugins/ldapusers.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/rpmutil.py | 231 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/simplerpm.py | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/svn.py | 106 | ||||
| -rwxr-xr-x[-rw-r--r--] | RepSys/util.py | 52 |
38 files changed, 975 insertions, 117 deletions
diff --git a/RepSys/ConfigParser.py b/RepSys/ConfigParser.py index 3b4e213..3b4e213 100644..100755 --- a/RepSys/ConfigParser.py +++ b/RepSys/ConfigParser.py diff --git a/RepSys/__init__.py b/RepSys/__init__.py index 2759e8f..b0df184 100644..100755 --- a/RepSys/__init__.py +++ b/RepSys/__init__.py @@ -9,6 +9,11 @@ config = ConfigParser.Config() tempfile.tempdir = config.get("global", "tempdir", None) or None # when "" del ConfigParser +def disable_mirror(*a, **kw): + config.set("global", "use-mirror", "no") + class Error(Exception): pass +class SilentError(Error): pass + # vim:et:ts=4:sw=4 diff --git a/RepSys/binrepo.py b/RepSys/binrepo.py new file mode 100755 index 0000000..edca9a4 --- /dev/null +++ b/RepSys/binrepo.py @@ -0,0 +1,393 @@ +from RepSys import Error, config, mirror, layout +from RepSys.util import execcmd, rellink +from RepSys.svn import SVN + +import sys +import os +import string +import stat +import shutil +import re +import tempfile +import hashlib +import urlparse +import threading +from cStringIO import StringIO + +DEFAULT_TARBALLS_REPO = "/tarballs" +BINARIES_DIR_NAME = "SOURCES" +BINARIES_CHECKOUT_NAME = "SOURCES-bin" + +PROP_USES_BINREPO = "mdv:uses-binrepo" +PROP_BINREPO_REV = "mdv:binrepo-rev" + +BINREPOS_SECTION = "binrepos" + +SOURCES_FILE = "sha1.lst" + +class ChecksumError(Error): + pass + +def svn_baseurl(target): + svn = SVN() + info = svn.info2(target) + if info is None: + # unversioned resource + newtarget = os.path.dirname(target) + info = svn.info2(newtarget) + assert info is not None, "svn_basedir should not be used with a "\ + "non-versioned directory" + root = info["Repository Root"] + url = info["URL"] + kind = info["Node Kind"] + path = url[len(root):] + if kind == "directory": + return url + basepath = os.path.dirname(path) + baseurl = mirror.normalize_path(url + "/" + basepath) + return baseurl + +def svn_root(target): + svn = SVN() + info = svn.info2(target) + if info is None: + newtarget = os.path.dirname(target) + info = svn.info2(newtarget) + assert info is not None + return info["Repository Root"] + +def enabled(url): + #TODO use information from url to find out whether we have a binrepo + # available for this url + use = config.getbool("global", "use-binaries-repository", False) + return use + +def default_repo(): + base = config.get("global", "binaries-repository", None) + if base is None: + default_parent = config.get("global", "default_parent", None) + if default_parent is None: + raise Error, "no binaries-repository nor default_parent "\ + "configured" + comps = urlparse.urlparse(default_parent) + base = comps[1] + ":" + DEFAULT_TARBALLS_REPO + return base + +def translate_url(url): + url = mirror.normalize_path(url) + main = mirror.normalize_path(layout.repository_url()) + subpath = url[len(main)+1:] + # [binrepos] + # updates/2009.0 = svn+ssh://svn.mandriva.com/svn/binrepo/20090/ + ## svn+ssh://svn.mandriva.com/svn/packages/2009.0/trafshow/current + ## would translate to + ## svn+ssh://svn.mandriva.com/svn/binrepo/20090/updates/trafshow/current/ + binbase = None + if BINREPOS_SECTION in config.sections(): + for option, value in config.walk(BINREPOS_SECTION): + if subpath.startswith(option): + binbase = value + break + binurl = mirror._joinurl(binbase or default_repo(), subpath) + return binurl + +def translate_topdir(path): + """Returns the URL in the binrepo from a given path inside a SVN + checkout directory. + + @path: if specified, returns a URL in the binrepo whose path is the + same as the path inside the main repository. + """ + baseurl = svn_baseurl(path) + binurl = translate_url(baseurl) + target = mirror.normalize_path(binurl) + return target + +def is_binary(path): + raw = config.get("binrepo", "upload-match", + "\.(7z|Z|bin|bz2|cpio|db|deb|egg|gem|gz|jar|jisp|lzma|"\ + "pdf|pgn\\.gz|pk3|rpm|rpm|run|sdz|smzip|tar|tbz|"\ + "tbz2|tgz|ttf|uqm|wad|war|xar|xpi|zip)$") + maxsize = config.getint("binrepo", "upload-match-size", "1048576") # 1MiB + expr = re.compile(raw) + name = os.path.basename(path) + if expr.search(name): + return True + st = os.stat(path) + if st[stat.ST_SIZE] >= maxsize: + return True + return False + +def find_binaries(paths): + new = [] + for path in paths: + if os.path.isdir(path): + for name in os.listdir(path): + fpath = os.path.join(path, name) + if is_binary(fpath): + new.append(fpath) + else: + if is_binary(path): + new.append(path) + return new + +def make_symlinks(source, dest): + todo = [] + tomove = [] + for name in os.listdir(source): + path = os.path.join(source, name) + if not os.path.isdir(path) and not name.startswith("."): + destpath = os.path.join(dest, name) + linkpath = rellink(path, destpath) + if os.path.exists(destpath): + if (os.path.islink(destpath) and + os.readlink(destpath) == linkpath): + continue + movepath = destpath + ".repsys-moved" + if os.path.exists(movepath): + raise Error, "cannot create symlink, %s already "\ + "exists (%s too)" % (destpath, movepath) + tomove.append((destpath, movepath)) + todo.append((destpath, linkpath)) + for destpath, movepath in tomove: + os.rename(destpath, movepath) + for destpath, linkpath in todo: + os.symlink(linkpath, destpath) + +def download(targetdir, pkgdirurl=None, export=False, show=True, + revision=None, symlinks=True, check=False): + assert not export or (export and pkgdirurl) + svn = SVN() + sourcespath = os.path.join(targetdir, "SOURCES") + binpath = os.path.join(targetdir, BINARIES_CHECKOUT_NAME) + if pkgdirurl: + topurl = translate_url(pkgdirurl) + else: + topurl = translate_topdir(targetdir) + binrev = None + if revision: + if pkgdirurl: + binrev = mapped_revision(pkgdirurl, revision) + else: + binrev = mapped_revision(targetdir, revision, wc=True) + binurl = mirror._joinurl(topurl, BINARIES_DIR_NAME) + if export: + svn.export(binurl, binpath, rev=binrev, show=show) + else: + svn.checkout(binurl, binpath, rev=binrev, show=show) + if symlinks: + make_symlinks(binpath, sourcespath) + if check: + check_sources(targetdir) + +def import_binaries(topdir, pkgname): + """Import all binaries from a given package checkout + + (with pending svn adds) + + @topdir: the path to the svn checkout + """ + svn = SVN() + topurl = translate_topdir(topdir) + sourcesdir = os.path.join(topdir, "SOURCES") + bintopdir = tempfile.mktemp("repsys") + try: + svn.checkout(topurl, bintopdir) + checkout = True + except Error: + bintopdir = tempfile.mkdtemp("repsys") + checkout = False + try: + bindir = os.path.join(bintopdir, BINARIES_DIR_NAME) + if not os.path.exists(bindir): + if checkout: + svn.mkdir(bindir) + else: + os.mkdir(bindir) + binaries = find_binaries([sourcesdir]) + update = update_sources_threaded(topdir, added=binaries) + for path in binaries: + name = os.path.basename(path) + binpath = os.path.join(bindir, name) + os.rename(path, binpath) + try: + svn.remove(path) + except Error: + # file not tracked + svn.revert(path) + if checkout: + svn.add(binpath) + log = "imported binaries for %s" % pkgname + if checkout: + rev = svn.commit(bindir, log=log) + else: + rev = svn.import_(bintopdir, topurl, log=log) + svn.propset(PROP_USES_BINREPO, "yes", topdir) + svn.propset(PROP_BINREPO_REV, str(rev), topdir) + update.join() + svn.add(sources_path(topdir)) + finally: + shutil.rmtree(bintopdir) + +def create_package_dirs(bintopdir): + svn = SVN() + binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME) + silent = config.get("log", "ignore-string", "SILENT") + message = "%s: created binrepo package structure" % silent + svn.mkdir(binurl, log=message, parents=True) + +def parse_sources(path): + entries = {} + f = open(path) + for rawline in f: + line = rawline.strip() + try: + sum, name = line.split(None, 1) + except ValueError: + # failed to unpack, line format error + raise Error, "invalid line in sources file: %s" % rawline + entries[name] = sum + return entries + +def check_hash(path, sum): + newsum = file_hash(path) + if newsum != sum: + raise ChecksumError, "different checksums for %s: expected %s, "\ + "but %s was found" % (path, sum, newsum) + +def check_sources(topdir): + spath = sources_path(topdir) + if not os.path.exists(spath): + raise Error, "'%s' was not found" % spath + entries = parse_sources(spath) + for name, sum in entries.iteritems(): + fpath = os.path.join(topdir, "SOURCES", name) + check_hash(fpath, sum) + +def file_hash(path): + sum = hashlib.sha1() + f = open(path) + while True: + block = f.read(4096) + if not block: + break + sum.update(block) + f.close() + return sum.hexdigest() + +def sources_path(topdir): + path = os.path.join(topdir, "SOURCES", SOURCES_FILE) + return path + +def update_sources(topdir, added=[], removed=[]): + path = sources_path(topdir) + entries = {} + if os.path.isfile(path): + entries = parse_sources(path) + f = open(path, "w") # open before calculating hashes + for name in removed: + entries.pop(removed) + for added_path in added: + name = os.path.basename(added_path) + entries[name] = file_hash(added_path) + for name in sorted(entries): + f.write("%s %s\n" % (entries[name], name)) + f.close() + +def update_sources_threaded(*args, **kwargs): + t = threading.Thread(target=update_sources, args=args, kwargs=kwargs) + t.start() + return t + +def upload(path, message=None): + from RepSys.rpmutil import getpkgtopdir + svn = SVN() + if not os.path.exists(path): + raise Error, "not found: %s" % path + # XXX check if the path is under SOURCES/ + paths = find_binaries([path]) + if not paths: + raise Error, "'%s' does not seem to have any tarballs" % path + topdir = getpkgtopdir() + bintopdir = translate_topdir(topdir) + binurl = mirror._joinurl(bintopdir, BINARIES_DIR_NAME) + sourcesdir = os.path.join(topdir, "SOURCES") + bindir = os.path.join(topdir, BINARIES_CHECKOUT_NAME) + silent = config.get("log", "ignore-string", "SILENT") + if not os.path.exists(bindir): + try: + download(topdir, show=False) + except Error: + # possibly the package does not exist + # (TODO check whether it is really a 'path not found' error) + pass + if not os.path.exists(bindir): + create_package_dirs(bintopdir) + svn.propset(PROP_USES_BINREPO, "yes", topdir) + svn.commit(topdir, log="%s: created binrepo structure" % silent) + download(topdir, show=False) + for path in paths: + if svn.info2(path): + sys.stderr.write("'%s' is already tracked by svn, ignoring\n" % + path) + continue + name = os.path.basename(path) + binpath = os.path.join(bindir, name) + os.rename(path, binpath) + svn.add(binpath) + if not message: + message = "%s: new binary files %s" % (silent, " ".join(paths)) + make_symlinks(bindir, sourcesdir) + update = update_sources_threaded(topdir, added=paths) + rev = svn.commit(binpath, log=message) + svn.propset(PROP_BINREPO_REV, str(rev), topdir) + sources = sources_path(topdir) + svn.add(sources) + update.join() + svn.commit(topdir + " " + sources, log=message, nonrecursive=True) + +def mapped_revision(target, revision, wc=False): + """Maps a txtrepo revision to a binrepo datespec + + This datespec can is intended to be used by svn .. -r DATE. + + @target: a working copy path or a URL + @revision: if target is a URL, the revision number used when fetching + svn info + @wc: if True indicates that 'target' must be interpreted as a + the path of a svn working copy, otherwise it is handled as a URL + """ + svn = SVN() + binrev = None + if wc: + spath = sources_path(target) + if os.path.exists(spath): + infolines = svn.info(spath, xml=True) + if infolines: + rawinfo = "".join(infolines) # arg! + found = re.search("<date>(.*?)</date>", rawinfo).groups() + date = found[0] + else: + raise Error, "bogus 'svn info' for '%s'" % spath + else: + raise Error, "'%s' was not found" % spath + else: + url = mirror._joinurl(target, sources_path("")) + date = svn.propget("svn:date", url, rev=revision, revprop=True) + if not date: + raise Error, "no valid date available for '%s'" % url + binrev = "{%s}" % date + return binrev + +def markrelease(sourceurl, releasesurl, version, release, revision): + svn = SVN() + binrev = mapped_revision(sourceurl, revision) + binsource = translate_url(sourceurl) + binreleases = translate_url(releasesurl) + versiondir = mirror._joinurl(binreleases, version) + dest = mirror._joinurl(versiondir, release) + svn.mkdir(binreleases, noerror=1, log="created directory for releases") + svn.mkdir(versiondir, noerror=1, log="created directory for version %s" % version) + svn.copy(binsource, dest, rev=binrev, + log="%%markrelease ver=%s rel=%s rev=%s binrev=%s" % (version, release, + revision, binrev)) diff --git a/RepSys/cgi/__init__.py b/RepSys/cgi/__init__.py index e69de29..e69de29 100644..100755 --- a/RepSys/cgi/__init__.py +++ b/RepSys/cgi/__init__.py diff --git a/RepSys/cgi/soapserver.py b/RepSys/cgi/soapserver.py index 2f6b751..2f6b751 100644..100755 --- a/RepSys/cgi/soapserver.py +++ b/RepSys/cgi/soapserver.py diff --git a/RepSys/cgi/submit.py b/RepSys/cgi/submit.py index 10f7cb2..10f7cb2 100644..100755 --- a/RepSys/cgi/submit.py +++ b/RepSys/cgi/submit.py diff --git a/RepSys/cgi/xmlrpcserver.py b/RepSys/cgi/xmlrpcserver.py index e0851d1..e0851d1 100644..100755 --- a/RepSys/cgi/xmlrpcserver.py +++ b/RepSys/cgi/xmlrpcserver.py diff --git a/RepSys/cgiutil.py b/RepSys/cgiutil.py index 35c5efb..35c5efb 100644..100755 --- a/RepSys/cgiutil.py +++ b/RepSys/cgiutil.py diff --git a/RepSys/command.py b/RepSys/command.py index f1d61f7..63f2df9 100644..100755 --- a/RepSys/command.py +++ b/RepSys/command.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from RepSys import Error, config +from RepSys import SilentError, Error, config import sys, os import urlparse import optparse @@ -37,6 +37,8 @@ def do_command(parse_options_func, main_func): try: opt = parse_options_func() main_func(**opt.__dict__) + except SilentError: + sys.exit(1) except Error, e: sys.stderr.write("error: %s\n" % str(e)) sys.exit(1) diff --git a/RepSys/commands/__init__.py b/RepSys/commands/__init__.py index e69de29..e69de29 100644..100755 --- a/RepSys/commands/__init__.py +++ b/RepSys/commands/__init__.py diff --git a/RepSys/commands/authoremail.py b/RepSys/commands/authoremail.py index f5b8b70..f5b8b70 100644..100755 --- a/RepSys/commands/authoremail.py +++ b/RepSys/commands/authoremail.py diff --git a/RepSys/commands/changed.py b/RepSys/commands/changed.py index 7d05604..66c1a53 100644..100755 --- a/RepSys/commands/changed.py +++ b/RepSys/commands/changed.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from RepSys import Error +from RepSys import Error, disable_mirror from RepSys.command import * from RepSys.layout import package_url from RepSys.rpmutil import check_changed @@ -26,6 +26,8 @@ def parse_options(): parser = OptionParser(help=HELP) parser.add_option("-a", dest="all", action="store_true") parser.add_option("-s", dest="show", action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" diff --git a/RepSys/commands/ci.py b/RepSys/commands/ci.py index 8d373b5..8d373b5 100644..100755 --- a/RepSys/commands/ci.py +++ b/RepSys/commands/ci.py diff --git a/RepSys/commands/co.py b/RepSys/commands/co.py index 5349049..81e4140 100644..100755 --- a/RepSys/commands/co.py +++ b/RepSys/commands/co.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from RepSys import Error +from RepSys import Error, disable_mirror from RepSys.command import * from RepSys.rpmutil import checkout import getopt @@ -19,7 +19,11 @@ Options: -d The distribution branch to checkout from -b The package branch -r REV Revision to checkout + -S Do not download sources from the binaries repository + -L Do not make symlinks of the binaries downloaded in SOURCES/ + -s Only checkout the SPECS/ directory -M Do not use the mirror (use the main repository) + --check Check integrity of files fetched from the binary repository -h Show this message Examples: @@ -33,8 +37,18 @@ Examples: def parse_options(): parser = OptionParser(help=HELP) parser.add_option("-r", dest="revision") + parser.add_option("-S", dest="use_binrepo", default=True, + action="store_false") + parser.add_option("--check", dest="binrepo_check", default=False, + action="store_true") + parser.add_option("-L", dest="binrepo_link", default=True, + action="store_false") parser.add_option("--distribution", "-d", dest="distro", default=None) parser.add_option("--branch", "-b", dest="branch", default=None) + parser.add_option("-s", "--spec", dest="spec", default=False, + action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) opts, args = parser.parse_args() if len(args) not in (1, 2): raise Error, "invalid arguments" diff --git a/RepSys/commands/create.py b/RepSys/commands/create.py index ded8abe..ded8abe 100644..100755 --- a/RepSys/commands/create.py +++ b/RepSys/commands/create.py diff --git a/RepSys/commands/del.py b/RepSys/commands/del.py new file mode 100755 index 0000000..2c6902e --- /dev/null +++ b/RepSys/commands/del.py @@ -0,0 +1,30 @@ +from RepSys import Error +from RepSys.command import * +from RepSys.rpmutil import binrepo_delete + +HELP = """\ +Usage: repsys del [OPTIONS] [PATH] + +Remove a given file from the binary sources repository. + +Changes in the sources file will be left uncommited. + +Options: + -c automatically commit the 'sources' file + -h help + +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-c", dest="commit", default=False, + action="store_true") + opts, args = parser.parse_args() + if len(args): + opts.paths = args + else: + raise Error, "you need to provide a path" + return opts + +def main(): + do_command(parse_options, binrepo_delete) diff --git a/RepSys/commands/editlog.py b/RepSys/commands/editlog.py index 9d1afc5..9d1afc5 100644..100755 --- a/RepSys/commands/editlog.py +++ b/RepSys/commands/editlog.py diff --git a/RepSys/commands/getspec.py b/RepSys/commands/getspec.py index 6a8f7ea..a357ef9 100644..100755 --- a/RepSys/commands/getspec.py +++ b/RepSys/commands/getspec.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from RepSys import Error +from RepSys import Error, disable_mirror from RepSys.command import * from RepSys.layout import package_url from RepSys.rpmutil import get_spec @@ -24,6 +24,8 @@ Examples: def parse_options(): parser = OptionParser(help=HELP) parser.add_option("-t", dest="targetdir", default=".") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" diff --git a/RepSys/commands/getsrpm.py b/RepSys/commands/getsrpm.py index 8cbe1f1..1767bb7 100644..100755 --- a/RepSys/commands/getsrpm.py +++ b/RepSys/commands/getsrpm.py @@ -3,7 +3,7 @@ # This program will extract given version/revision of the named package # from the Conectiva Linux repository system. # -from RepSys import Error, config +from RepSys import Error, config, disable_mirror from RepSys.command import * from RepSys.layout import package_url from RepSys.rpmutil import get_srpm @@ -32,6 +32,8 @@ Options: -T FILE Template to be used to generate the %changelog -M Do not use the mirror (use the main repository) -h Show this message + -S Do not download sources from the binary repository + --check Check integrity of files fetched from the binary repository --strict Check if the given revision contains changes in REPPKGURL Examples: @@ -76,6 +78,12 @@ def parse_options(): parser.add_option("-n", dest="revname", action="store_true") parser.add_option("-l", dest="svnlog", action="store_true") parser.add_option("-T", dest="template", type="string", default=None) + parser.add_option("-S", dest="use_binrepo", default=True, + action="store_false") + parser.add_option("--check", dest="binrepo_check", default=False, + action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) parser.add_option("--strict", dest="strict", default=False, action="store_true") opts, args = parser.parse_args() diff --git a/RepSys/commands/log.py b/RepSys/commands/log.py new file mode 100755 index 0000000..28df27d --- /dev/null +++ b/RepSys/commands/log.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +from RepSys import config, mirror, disable_mirror +from RepSys.command import * +from RepSys.layout import package_url, checkout_url +from RepSys.rpmutil import sync +from RepSys.util import execcmd +import sys +import os + +HELP = """\ +Usage: repsys log [OPTIONS] [PACKAGE] + +Shows the SVN log for a given package. + +Options: + -h Show this message + -v Show changed paths + -l LIMIT Limit of log entries to show + -r REV Show a specific revision + -M Do not use the mirror (use the main repository) + +Examples: + repsys log mutt + repsys log 2009.1/mutt +""" + +def parse_options(): + parser = OptionParser(help=HELP) + parser.add_option("-v", dest="verbose", action="store_true", + default=False) + parser.add_option("-l", "--limit", dest="limit", type="int", + default=None) + parser.add_option("-r", dest="revision", type="string", default=None) + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) + opts, args = parser.parse_args() + if len(args): + opts.pkgdirurl = package_url(args[0]) + else: + parser.error("log requires a package name") + return opts + +def svn_log(pkgdirurl, verbose=False, limit=None, revision=None): + mirror.info(pkgdirurl) + url = checkout_url(pkgdirurl) + svncmd = config.get("global", "svn-command", "svn") + args = [svncmd, "log", url] + if verbose: + args.append("-v") + if limit: + args.append("-l") + args.append(limit) + if revision: + args.append("-r") + args.append(revision) + if os.isatty(sys.stdin.fileno()): + args.append("| less") + rawcmd = " ".join(args) + execcmd(rawcmd, show=True) + +def main(): + do_command(parse_options, svn_log) diff --git a/RepSys/commands/markrelease.py b/RepSys/commands/markrelease.py index 057cf1d..057cf1d 100644..100755 --- a/RepSys/commands/markrelease.py +++ b/RepSys/commands/markrelease.py diff --git a/RepSys/commands/patchspec.py b/RepSys/commands/patchspec.py index 9a4881b..9a4881b 100644..100755 --- a/RepSys/commands/patchspec.py +++ b/RepSys/commands/patchspec.py diff --git a/RepSys/commands/putsrpm.py b/RepSys/commands/putsrpm.py index 751fa0b..efe1a15 100644..100755 --- a/RepSys/commands/putsrpm.py +++ b/RepSys/commands/putsrpm.py @@ -22,6 +22,7 @@ Options: -c URL The URL of the base directory where the changelog will be placed -s Don't strip the changelog from the spec + (nor import it into misc/) -n Don't try to rename the spec file -h Show this message diff --git a/RepSys/commands/rpmlog.py b/RepSys/commands/rpmlog.py index 11fe36d..238b675 100644..100755 --- a/RepSys/commands/rpmlog.py +++ b/RepSys/commands/rpmlog.py @@ -3,7 +3,7 @@ # This program will convert the output of "svn log" to be suitable # for usage in an rpm %changelog session. # -from RepSys import Error, layout +from RepSys import Error, layout, disable_mirror from RepSys.command import * from RepSys.svn import SVN from RepSys.log import get_changelog, split_spec_changelog @@ -43,6 +43,8 @@ def parse_options(): action="store_true") parser.add_option("-s", dest="sort", default=False, action="store_true") + parser.add_option("-M", "--no-mirror", action="callback", + callback=disable_mirror) opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" diff --git a/RepSys/commands/submit.py b/RepSys/commands/submit.py index 88ff596..2924329 100644..100755 --- a/RepSys/commands/submit.py +++ b/RepSys/commands/submit.py @@ -1,5 +1,6 @@ #!/usr/bin/python -from RepSys import Error, config, layout +from RepSys import Error, config, layout, mirror +from RepSys.svn import SVN from RepSys.command import * from RepSys.rpmutil import get_spec, get_submit_info from RepSys.util import get_auth, execcmd, get_helper @@ -8,6 +9,7 @@ import getopt import sys import re import subprocess +import uuid import xmlrpclib @@ -37,6 +39,8 @@ Options: argument) -s The host in which the package URL will be submitted (defaults to the host in the URL) + -a Submit all URLs at once (depends on server-side support) + -i SID Use the submit identifier SID -h Show this message --distro The distribution branch where the packages come from --define Defines one variable to be used by the submit scripts @@ -63,6 +67,9 @@ def parse_options(): parser.add_option("-r", dest="revision", type="string", nargs=1) parser.add_option("-s", dest="submithost", type="string", nargs=1, default=None) + parser.add_option("-i", dest="sid", type="string", nargs=1, + default=None) + parser.add_option("-a", dest="atonce", action="store_true", default=False) parser.add_option("--distro", dest="distro", type="string", default=None) parser.add_option("--define", action="append", default=[]) @@ -96,8 +103,33 @@ def parse_options(): if expanded != args: print "Submitting: %s" % " ".join(expanded) args = expanded - opts.urls = [layout.package_url(nameurl, distro=opts.distro, mirrored=False) + # generate URLs for package names: + opts.urls = [mirror.strip_username( + layout.package_url(nameurl, distro=opts.distro, mirrored=False)) for nameurl in args] + # find the revision if not specified: + newurls = [] + for url in opts.urls: + if not "@" in url: + print "Fetching revision..." + courl = layout.checkout_url(url) + log = SVN().log(courl, limit=1) + if not log: + raise Error, "can't find a revision for %s" % courl + ci = log[0] + print "URL:", url + print "Commit:", + print "%d | %s" % (ci.revision, ci.author), + if ci.lines: + line = " ".join(ci.lines).strip() + if len(line) > 57: + line = line[:57] + "..." + print "| %s" % line, + print + url = url + "@" + str(ci.revision) + newurls.append(url) + opts.urls[:] = newurls + # choose a target if not specified: if opts.target is None and opts.distro is None: target = layout.distro_branch(opts.urls[0]) or DEFAULT_TARGET print "Implicit target: %s" % target @@ -132,7 +164,7 @@ def list_targets(option, opt, val, parser): execcmd(command, show=True) sys.exit(0) -def submit(urls, target, define=[], submithost=None): +def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): if submithost is None: submithost = config.get("submit", "host") if submithost is None: @@ -145,25 +177,33 @@ def submit(urls, target, define=[], submithost=None): # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory createsrpm = get_helper("create-srpm") - args = ["ssh", submithost, createsrpm, "-t", target] - for entry in define: - args.append("--define") - args.append(entry) + baseargs = ["ssh", submithost, createsrpm, "-t", target] + if not sid: + sid = uuid.uuid4() + define.append("sid=%s" % sid) + for entry in reversed(define): + baseargs.append("--define") + baseargs.append(entry) + cmdsargs = [] if len(urls) == 1: # be compatible with server-side repsys versions older than 1.6.90 url, rev = layout.split_url_revision(urls[0]) - args.append(url) - args.append("-r") - args.append(str(rev)) - else: - args.extend(urls) - command = subprocess.list2cmdline(args) - status, output = execcmd(command) - if status == 0: - print "Package submitted!" + baseargs.append("-r") + baseargs.append(str(rev)) + baseargs.append(url) + cmdsargs.append(baseargs) + elif atonce: + cmdsargs.append(baseargs + urls) else: - sys.stderr.write(output) - sys.exit(status) + cmdsargs.extend((baseargs + [url]) for url in urls) + for cmdargs in cmdsargs: + command = subprocess.list2cmdline(cmdargs) + status, output = execcmd(command) + if status == 0: + print "Package submitted!" + else: + sys.stderr.write(output) + sys.exit(status) def main(): do_command(parse_options, submit) diff --git a/RepSys/commands/switch.py b/RepSys/commands/switch.py index 5cbe2d7..998ae2c 100644..100755 --- a/RepSys/commands/switch.py +++ b/RepSys/commands/switch.py @@ -7,8 +7,8 @@ Usage: repsys switch [URL] Relocates the working copy to the base location URL. -If URL is not provided, it will use the option default_parent from -repsys.conf as default, or, if the current working copy is already based in +If URL is not provided, it will use the option repository from repsys.conf +as default, or, if the current working copy is already based in default_parent, it will use the location from the mirror option from repsys.conf. @@ -19,7 +19,7 @@ Options: Examples: repsys switch - repsys switch https://mirrors.localnetwork/svn/packages/cooker + repsys switch https://mirrors.localnetwork/svn/packages/ """ def parse_options(): diff --git a/RepSys/commands/sync.py b/RepSys/commands/sync.py index a51db22..b4bdaba 100644..100755 --- a/RepSys/commands/sync.py +++ b/RepSys/commands/sync.py @@ -5,12 +5,13 @@ from RepSys.rpmutil import sync HELP = """\ Usage: repsys sync -Will add or removed from the working copy new files added or removed -from the spec file. +Will add or remove from the working copy those files added or removed +in the spec file. -"No changes are commited." +It will not commit the changes. Options: + -c Commit the changes, as in ci --dry-run Print results without changing the working copy --download -d Try to download the source files not found @@ -24,6 +25,8 @@ def parse_options(): parser = OptionParser(help=HELP) parser.add_option("--dry-run", dest="dryrun", default=False, action="store_true") + parser.add_option("-c", dest="ci", default=False, + action="store_true") parser.add_option("-d", "--download", dest="download", default=False, action="store_true") opts, args = parser.parse_args() diff --git a/RepSys/commands/up.py b/RepSys/commands/up.py new file mode 100755 index 0000000..02a1a9f --- /dev/null +++ b/RepSys/commands/up.py @@ -0,0 +1,22 @@ +from RepSys import Error +from RepSys.command import * +from RepSys.rpmutil import update + +HELP = """\ +Usage: repsys up [PATH] + +Update the package working copy and synchronize all binaries. + +Options: + -h help +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + if args: + opts.target = args[0] + return opts + +def main(): + do_command(parse_options, update) diff --git a/RepSys/commands/upload.py b/RepSys/commands/upload.py new file mode 100755 index 0000000..6af50ea --- /dev/null +++ b/RepSys/commands/upload.py @@ -0,0 +1,28 @@ +from RepSys import Error +from RepSys.command import * +from RepSys.rpmutil import upload + +HELP = """\ +Usage: repsys upload [OPTIONS] [PATH] + +Upload a given file to the binary sources repository. + +It will also update the contents of the 'binrepo.lst' file and leave it +uncommited. + +If the path is a directory, all the contents of the directory will be +uploaded or removed. + +Options: + -h help + +""" + +def parse_options(): + parser = OptionParser(help=HELP) + opts, args = parser.parse_args() + opts.paths = args + return opts + +def main(): + do_command(parse_options, upload) diff --git a/RepSys/layout.py b/RepSys/layout.py index a358c28..fb50acd 100644..100755 --- a/RepSys/layout.py +++ b/RepSys/layout.py @@ -117,7 +117,7 @@ def remove_current(pkgdirurl): def repository_url(mirrored=False): url = None - if mirrored and config.getbool("global", "use-mirror"): + if mirrored and config.getbool("global", "use-mirror", "yes"): url = config.get("global", "mirror") if url is None: url = config.get("global", "repository") diff --git a/RepSys/log.py b/RepSys/log.py index a1d1944..6cb9da1 100644..100755 --- a/RepSys/log.py +++ b/RepSys/log.py @@ -18,6 +18,7 @@ import locale import glob import tempfile import shutil +import subprocess locale.setlocale(locale.LC_ALL, "C") @@ -86,11 +87,15 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None): specpath = found[0] options = rpm_macros_defs(macros) command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' " - "--specfile %s %s 2>/dev/null") % + "--specfile %s %s") % (specpath, options)) - status, output = execcmd(command) - if status != 0: - raise Error, "Error in command %s: %s" % (command, output) + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + pipe.wait() + output = pipe.stdout.read() + error = pipe.stderr.read() + if pipe.returncode != 0: + raise Error, "Error in command %s: %s" % (command, error) releases = output.split() try: epoch, vr = releases[0].split(":", 1) @@ -514,18 +519,23 @@ def split_spec_changelog(stream): chlog = StringIO() spec = StringIO() found = 0 + visible = 0 for line in stream: if line.startswith("%changelog"): found = 1 elif not found: spec.write(line) elif found: + if line.strip(): + visible = 1 chlog.write(line) elif line.startswith("%"): found = 0 spec.write(line) spec.seek(0) - chlog.seek(0) + if not visible: + # when there are only blanks in the changelog, make it empty + chlog = StringIO() return spec, chlog def get_old_log(pkgdirurl): diff --git a/RepSys/mirror.py b/RepSys/mirror.py index f0d2316..94720cc 100644..100755 --- a/RepSys/mirror.py +++ b/RepSys/mirror.py @@ -66,9 +66,12 @@ def using_on(url): using = False return using -def info(url, stream=sys.stderr): +def info(url, write=False, stream=sys.stderr): if using_on(url): - stream.write("using mirror\n") + stream.write("Using the svn mirror.\n") + if write: + stream.write("To be able to commit changes, use " + "'repsys switch' first.\n") def mirror_relocate(oldparent, newparent, url, wcpath): svn = SVN() diff --git a/RepSys/plugins/__init__.py b/RepSys/plugins/__init__.py index e4f4e08..e4f4e08 100644..100755 --- a/RepSys/plugins/__init__.py +++ b/RepSys/plugins/__init__.py diff --git a/RepSys/plugins/ldapusers.py b/RepSys/plugins/ldapusers.py index e56371d..e56371d 100644..100755 --- a/RepSys/plugins/ldapusers.py +++ b/RepSys/plugins/ldapusers.py diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py index c11ad4e..bff744b 100644..100755 --- a/RepSys/rpmutil.py +++ b/RepSys/rpmutil.py @@ -1,6 +1,6 @@ #!/usr/bin/python from RepSys import Error, config -from RepSys import mirror, layout, log +from RepSys import mirror, layout, log, binrepo from RepSys.svn import SVN from RepSys.simplerpm import SRPM from RepSys.util import execcmd @@ -19,12 +19,16 @@ def get_spec(pkgdirurl, targetdir=".", submit=False): tmpdir = tempfile.mktemp() try: geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") + mirror.info(geturl) svn.export("'%s'" % geturl, tmpdir) speclist = glob.glob(os.path.join(tmpdir, "*.spec")) if not speclist: raise Error, "no spec files found" spec = speclist[0] shutil.copy(spec, targetdir) + name = os.path.basename(spec) + path = os.path.join(targetdir, name) + print "Wrote %s" % (name) finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) @@ -65,7 +69,9 @@ def get_srpm(pkgdirurl, template = None, macros = [], verbose = 0, - strict = False): + strict = False, + use_binrepo = False, + binrepo_check = True): svn = SVN() tmpdir = tempfile.mktemp() topdir = "--define '_topdir %s'" % tmpdir @@ -95,10 +101,19 @@ def get_srpm(pkgdirurl, "inside %s" % (revision or "HEAD", geturl) mirror.info(geturl) svn.export(geturl, tmpdir, rev=revision) + if use_binrepo: + binrepo_check = (binrepo_check or + config.getbool("binrepo", "getsrpm-check", False)) + download_binaries(tmpdir, geturl, revision=revision, + export=True, check=binrepo_check) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) + if config.getbool("srpm", "run-prep", False): + makefile = os.path.join(tmpdir, "Makefile") + if os.path.exists(makefile): + execcmd("make", "-C", tmpdir, "srpm-prep") if not speclist: raise Error, "no spec files found" spec = speclist[0] @@ -285,29 +300,32 @@ def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, specpath = specpath fspec = open(specpath) spec, chlog = log.split_spec_changelog(fspec) - chlog.seek(0) - spec.seek(0) fspec.close() fspec = open(specpath, "w") fspec.writelines(spec) fspec.close() - oldurl = baseold or config.get("log", "oldurl") - pkgoldurl = mirror._joinurl(oldurl, srpm.name) - svn.mkdir(pkgoldurl, noerror=1, - log="created old log directory for %s" % srpm.name) - logtmp = tempfile.mktemp() - try: - svn.checkout(pkgoldurl, logtmp) - miscpath = os.path.join(logtmp, "log") - fmisc = open(miscpath, "w+") - fmisc.writelines(chlog) - fmisc.close() - svn.add(miscpath) - svn.commit(logtmp, - log="imported old log for %s" % srpm.name) - finally: - if os.path.isdir(logtmp): - shutil.rmtree(logtmp) + chlog.seek(0, os.SEEK_END) + if chlog.tell() != 0: + chlog.seek(0) + #FIXME move it to layout.py + oldurl = baseold or config.get("log", "oldurl") + pkgoldurl = mirror._joinurl(oldurl, srpm.name) + svn.mkdir(pkgoldurl, noerror=1, + log="created old log directory for %s" % srpm.name) + logtmp = tempfile.mktemp() + try: + svn.checkout(pkgoldurl, logtmp) + miscpath = os.path.join(logtmp, "log") + fmisc = open(miscpath, "w+") + fmisc.writelines(chlog) + fmisc.close() + svn.add(miscpath) + svn.commit(logtmp, + log="imported old log for %s" % srpm.name) + finally: + if os.path.isdir(logtmp): + shutil.rmtree(logtmp) + binrepo.import_binaries(currentdir, srpm.name) svn.commit(tmpdir, log=logmsg or ("imported package %s" % srpm.name)) finally: @@ -370,9 +388,11 @@ revision: %s def mark_release(pkgdirurl, version, release, revision): svn = SVN() - releasesurl = "/".join([pkgdirurl, "releases"]) + releasesurl = layout.checkout_url(pkgdirurl, releases=True) versionurl = "/".join([releasesurl, version]) releaseurl = "/".join([versionurl, release]) + currenturl = layout.checkout_url(pkgdirurl) + binrepo.markrelease(currenturl, releasesurl, version, release, revision) if svn.ls(releaseurl, noerror=1): raise Error, "release already exists" svn.mkdir(releasesurl, noerror=1, @@ -382,7 +402,6 @@ def mark_release(pkgdirurl, version, release, revision): pristineurl = layout.checkout_url(pkgdirurl, pristine=True) svn.remove(pristineurl, noerror=1, log="Removing previous pristine/ directory.") - currenturl = layout.checkout_url(pkgdirurl) svn.copy(currenturl, pristineurl, log="Copying release %s-%s to pristine/ directory." % (version, release)) @@ -450,34 +469,51 @@ def check_changed(pkgdirurl, all=0, show=0, verbose=0): "nocurrent": nocurrent, "nopristine": nopristine} -def checkout(pkgdirurl, path=None, revision=None, branch=None, - distro=None): +def checkout(pkgdirurl, path=None, revision=None, branch=None, distro=None, + spec=False, use_binrepo=False, binrepo_check=True, binrepo_link=True): o_pkgdirurl = pkgdirurl pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro) - current = layout.checkout_url(pkgdirurl, branch=branch) + append = None + if spec: + append = "SPECS" + current = layout.checkout_url(pkgdirurl, branch=branch, + append_path=append) if path is None: path = layout.package_name(pkgdirurl) - mirror.info(current) + mirror.info(current, write=True) svn = SVN() svn.checkout(current, path, rev=revision, show=1) - -def _getpkgtopdir(basedir=None): + if use_binrepo: + download_binaries(path, revision=revision, symlinks=binrepo_link, + check=binrepo_check) + +def getpkgtopdir(basedir=None): + #FIXME this implementation doesn't work well with relative path names, + # which is something we need in order to have a friendlier output if basedir is None: - basedir = os.getcwd() - cwd = os.getcwd() - dirname = os.path.basename(cwd) - if dirname == "SPECS" or dirname == "SOURCES": - topdir = os.pardir - else: - topdir = "" - return topdir + basedir = os.path.curdir + while not ispkgtopdir(basedir): + if os.path.abspath(basedir) == "/": + raise Error, "can't find top package directories SOURCES and SPECS" + basedir = os.path.join(basedir, os.path.pardir) + if basedir.startswith("./"): + basedir = basedir[2:] + return basedir + +def ispkgtopdir(path=None): + if path is None: + path = os.getcwd() + names = os.listdir(path) + return (".svn" in names and "SPECS" in names and "SOURCES" in names) -def sync(dryrun=False, download=False): +def sync(dryrun=False, ci=False, download=False): + # TODO FIXME XXX fix it! + raise Error, "sync is not expected to work these days" svn = SVN() - topdir = _getpkgtopdir() + topdir = getpkgtopdir() # run svn info because svn st does not complain when topdir is not an # working copy - svn.info(topdir or ".") + svn.info(topdir) specsdir = os.path.join(topdir, "SPECS/") sourcesdir = os.path.join(topdir, "SOURCES/") for path in (specsdir, sourcesdir): @@ -496,15 +532,28 @@ def sync(dryrun=False, download=False): for name, no, flags in spec.sources()) sourcesst = dict((os.path.basename(path), (path, st)) for st, path in svn.status(sourcesdir, noignore=True)) - toadd = [] + toadd_br = [] + toadd_svn = [] + toremove_svn = [] + toremove_br = [] + # add the spec file itself, in case of a new package + specstl = svn.status(specpath, noignore=True) + if specstl: + specst, _ = specstl[0] + if specst == "?": + toadd_svn.append(specpath) + # add source files: for source, url in sources.iteritems(): sourcepath = os.path.join(sourcesdir, source) - pst = sourcesst.get(source) - if pst: - if os.path.isfile(sourcepath): - toadd.append(sourcepath) + if sourcesst.get(source): + if not os.path.islink(sourcepath): + if not binrepo.is_tracked(sourcepath): + if binrepo.is_binary(sourcepath): + toadd_br.append(sourcepath) + else: + toadd_svn.append(sourcepath) else: - sys.stderr.write("warning: %s not found, skipping\n" % sourcepath) + sys.stderr.write("warning: %s not found\n" % sourcepath) elif download and not os.path.isfile(sourcepath): print "%s not found, downloading from %s" % (sourcepath, url) fmt = config.get("global", "download-command", @@ -517,29 +566,47 @@ def sync(dryrun=False, download=False): "configuration option" % e execcmd(cmd, show=True) if os.path.isfile(sourcepath): - toadd.append(sourcepath) + if binrepo.is_binary(sourcepath): + toadd_br.append(sourcepath) + else: + toadd_svn.append(sourcepath) else: raise Error, "file not found: %s" % sourcepath # rm entries not found in sources and still in svn found = os.listdir(sourcesdir) - toremove = [] for entry in found: - if entry == ".svn": + if entry == ".svn" or entry == "sources": continue status = sourcesst.get(entry) - if status is None and entry not in sources: - path = os.path.join(sourcesdir, entry) - toremove.append(path) - for path in toremove: + path = os.path.join(sourcesdir, entry) + if entry not in sources: + if status is None: # file is tracked by svn + toremove_svn.append(path) + elif binrepo.is_tracked(path): + toremove_br.append(path) + for path in toremove_svn: print "D\t%s" % path if not dryrun: svn.remove(path, local=True) - for path in toadd: + for path in toremove_br: + print "DB\t%s" % path + if not dryrun: + binrepo.delete_pending(path) + for path in toadd_svn: print "A\t%s" % path if not dryrun: svn.add(path, local=True) + for path in toadd_br: + print "AB\t%s" % path + if not dryrun: + binrepo.upload_pending(path) + if commit: + commit(topdir) def commit(target=".", message=None, logfile=None): + topdir = getpkgtopdir(target) + sourcesdir = os.path.join(topdir, "SOURCES") + binrepo.commit(sourcesdir) #TODO make it optional svn = SVN() status = svn.status(target, quiet=True) if not status: @@ -566,9 +633,63 @@ def commit(target=".", message=None, logfile=None): print "use \"repsys switch\" in order to switch back to mirror "\ "later" +def spec_sources(topdir): + specs = glob.glob(os.path.join(topdir, "SPECS/*.spec")) + spec_path = specs[0] # FIXME use svn info to ensure which one + ts = rpm.ts() + spec = ts.parseSpec(spec_path) + sources = [name for name, x, y in spec.sources()] + return sources + +def download_binaries(target, pkgdirurl=None, export=False, revision=None, + symlinks=True, check=False): + refurl = pkgdirurl + if refurl is None: + refurl = binrepo.svn_root(target) + if binrepo.enabled(refurl): + binrepo.download(target, pkgdirurl, export=export, + revision=revision, symlinks=symlinks, check=check) + +def update(target=None): + svn = SVN() + info = None + svn_target = None + br_target = None + if target: + svn_target = target + else: + top = getpkgtopdir() + svn_target = top + br_target = top + if svn_target: + svn.update(svn_target, show=True) + if br_target: + info = svn.info2(svn_target) + if not br_target and not svn_target: + raise Error, "target not in SVN nor in binaries "\ + "repository: %s" % target + url = info["URL"] + download_binaries(br_target, url) + +def upload(paths): + for path in paths: + binrepo.upload(path) + +def binrepo_delete(paths, commit=False): + #TODO handle files tracked by svn + refurl = binrepo.svn_root(paths[0]) + if not binrepo.enabled(refurl): + raise Error, "binary repository is not enabled for %s" % refurl + added, deleted = binrepo.remove(paths) + if commit: + svn = SVN() + spath = binrepo.sources_path(paths[0]) + log = _sources_log(added, deleted) + svn.commit(spath, log=log) + def switch(mirrorurl=None): svn = SVN() - topdir = _getpkgtopdir() + topdir = getpkgtopdir() info = svn.info2(topdir) wcurl = info.get("URL") if wcurl is None: diff --git a/RepSys/simplerpm.py b/RepSys/simplerpm.py index d448c5f..d448c5f 100644..100755 --- a/RepSys/simplerpm.py +++ b/RepSys/simplerpm.py diff --git a/RepSys/svn.py b/RepSys/svn.py index 985329d..d6be524 100644..100755 --- a/RepSys/svn.py +++ b/RepSys/svn.py @@ -1,6 +1,7 @@ -from RepSys import Error, config +from RepSys import Error, SilentError, config from RepSys.util import execcmd, get_auth import sys +import os import re import time @@ -23,23 +24,63 @@ class SVN: if not kwargs.get("show") and args[0] not in localcmds: args = list(args) args.append("--non-interactive") - svn_command = config.get("global", "svn-command", - "SVN_SSH='ssh -o \"BatchMode yes\"' svn") + else: + kwargs["geterr"] = True + kwargs["cleanerr"] = True + if kwargs.get("xml"): + args.append("--xml") + self._set_env() + svn_command = config.get("global", "svn-command", "svn") cmdstr = svn_command + " " + " ".join(args) try: return execcmd(cmdstr, **kwargs) except Error, e: - if "Permission denied" in e.message: - raise Error, ("%s\n" - "Seems ssh-agent or ForwardAgent are not setup, see " - "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" - " for more information." % e) - elif "authorization failed" in e.message: - raise Error, ("%s\n" - "Note that repsys does not support any HTTP " - "authenticated access." % e) + msg = None + if e.args: + if "Permission denied" in e.args[0]: + msg = ("It seems ssh-agent or ForwardAgent are not setup " + "or your username is wrong. See " + "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" + " for more information.") + elif "authorization failed" in e.args[0]: + msg = ("Note that repsys does not support any HTTP " + "authenticated access.") + if kwargs.get("show") and \ + not config.getbool("global", "verbose", 0): + # svn has already dumped error messages, we don't need to + # do it too + if msg: + sys.stderr.write("\n") + sys.stderr.write(msg) + sys.stderr.write("\n") + raise SilentError + elif msg: + raise Error, "%s\n%s" % (e, msg) raise + def _set_env(self): + wrapper = "repsys-ssh" + repsys = config.get("global", "repsys-cmd") + if repsys: + dir = os.path.dirname(repsys) + path = os.path.join(dir, wrapper) + if os.path.exists(path): + wrapper = path + defaults = {"SVN_SSH": wrapper} + os.environ.update(defaults) + raw = config.get("global", "svn-env") + if raw: + for line in raw.split("\n"): + env = line.strip() + if not env: + continue + try: + name, value = env.split("=", 1) + except ValueError: + sys.stderr.write("invalid svn environment line: %r\n" % env) + continue + os.environ[name] = value + def _execsvn_success(self, *args, **kwargs): status, output = self._execsvn(*args, **kwargs) return status == 0 @@ -59,12 +100,13 @@ class SVN: if not optional or received_kwargs.has_key("rev"): ret = received_kwargs.get("rev") if isinstance(ret, basestring): - try: - ret = int(ret) - except ValueError: - raise Error, "invalid revision provided" + if not ret.startswith("{"): # if not a datespec + try: + ret = int(ret) + except ValueError: + raise Error, "invalid revision provided" if ret: - cmd_args.append("-r %d" % ret) + cmd_args.append("-r '%s'" % ret) def add(self, path, **kwargs): cmd = ["add", path] @@ -85,13 +127,29 @@ class SVN: def mkdir(self, path, **kwargs): cmd = ["mkdir", path] + if kwargs.get("parents"): + cmd.append("--parents") self._add_log(cmd, kwargs) return self._execsvn_success(*cmd, **kwargs) + def _execsvn_commit(self, *cmd, **kwargs): + status, output = self._execsvn(*cmd, **kwargs) + match = re.search("Committed revision (?P<rev>\\d+)\\.$", output) + if match: + rawrev = match.group("rev") + return int(rawrev) + def commit(self, path, **kwargs): cmd = ["commit", path] + if kwargs.get("nonrecursive"): + cmd.append("-N") self._add_log(cmd, kwargs) - return self._execsvn_success(*cmd, **kwargs) + return self._execsvn_commit(*cmd, **kwargs) + + def import_(self, path, url, **kwargs): + cmd = ["import", "'%s'" % path, "'%s'" % url] + self._add_log(cmd, kwargs) + return self._execsvn_commit(*cmd, **kwargs) def export(self, url, targetpath, **kwargs): cmd = ["export", "'%s'" % url, targetpath] @@ -102,6 +160,14 @@ class SVN: cmd = ["checkout", "'%s'" % url, targetpath] self._add_revision(cmd, kwargs, optional=1) return self._execsvn_success(*cmd, **kwargs) + + def propget(self, propname, targets, **kwargs): + cmd = ["propget", propname, targets] + if kwargs.get("revprop"): + cmd.append("--revprop") + self._add_revision(cmd, kwargs) + status, output = self._execsvn(local=True, *cmd, **kwargs) + return output def propset(self, propname, value, targets, **kwargs): cmd = ["propset", propname, "'%s'" % value, targets] @@ -125,8 +191,8 @@ class SVN: def info(self, path, **kwargs): cmd = ["info", path] - status, output = self._execsvn(local=True, *cmd, **kwargs) - if status == 0 and "Not a versioned resource" not in output: + status, output = self._execsvn(local=True, noerror=True, *cmd, **kwargs) + if "Not a versioned resource" not in output: return output.splitlines() return None diff --git a/RepSys/util.py b/RepSys/util.py index 83f2ebe..84840b9 100644..100755 --- a/RepSys/util.py +++ b/RepSys/util.py @@ -2,11 +2,13 @@ from RepSys import Error, config +import subprocess import getpass import sys import os import re import logging +from cStringIO import StringIO #import commands log = logging.getLogger("repsys") @@ -26,14 +28,35 @@ def commands_getstatusoutput(cmd): def execcmd(*cmd, **kwargs): cmdstr = " ".join(cmd) if kwargs.get("show"): - status = os.system(cmdstr) - output = "" + if kwargs.get("geterr"): + err = StringIO() + pipe = subprocess.Popen(cmdstr, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + of = pipe.stdout.fileno() + ef = pipe.stderr.fileno() + while True: + odata = os.read(of, 8192) + sys.stdout.write(odata) + edata = os.read(ef, 8192) + err.write(edata) + sys.stderr.write(edata) + status = pipe.poll() + if status is not None and not (odata and edata): + break + output = err.getvalue() + else: + status = os.system(cmdstr) + output = "" else: status, output = commands_getstatusoutput( "LANG=C LANGUAGE=C LC_ALL=C "+cmdstr) + verbose = config.getbool("global", "verbose", 0) if status != 0 and not kwargs.get("noerror"): - raise Error, "command failed: %s\n%s\n" % (cmdstr, output) - if config.getbool("global", "verbose", 0): + if kwargs.get("cleanerr") and not verbose: + raise Error, output + else: + raise Error, "command failed: %s\n%s\n" % (cmdstr, output) + if verbose: print cmdstr sys.stdout.write(output) return status, output @@ -92,6 +115,27 @@ def get_helper(name): if not os.path.isfile(hpath): log.warn("providing unexistent helper: %s", hpath) return hpath + +def rellink(src, dst): + """Creates relative symlinks + + It will find the common ancestor and append to the src path. + """ + asrc = os.path.abspath(src) + adst = os.path.abspath(dst) + csrc = asrc.split(os.path.sep) + cdst = adst.split(os.path.sep) + dstname = cdst.pop() + i = 0 + l = min(len(csrc), len(cdst)) + while i < l: + if csrc[i] != cdst[i]: + break + i += 1 + dstextra = len(cdst[i:]) + steps = [os.path.pardir] * dstextra + steps.extend(csrc[i:]) + return os.path.sep.join(steps) # vim:et:ts=4:sw=4 |