diff options
Diffstat (limited to 'RepSys')
| -rw-r--r-- | RepSys/ConfigParser.py | 5 | ||||
| -rw-r--r-- | RepSys/__init__.py | 16 | ||||
| -rw-r--r-- | RepSys/command.py | 12 | ||||
| -rw-r--r-- | RepSys/commands/authoremail.py | 3 | ||||
| -rw-r--r-- | RepSys/commands/changed.py | 6 | ||||
| -rw-r--r-- | RepSys/commands/ci.py | 12 | ||||
| -rw-r--r-- | RepSys/commands/co.py | 20 | ||||
| -rw-r--r-- | RepSys/commands/create.py | 8 | ||||
| -rw-r--r-- | RepSys/commands/editlog.py | 6 | ||||
| -rw-r--r-- | RepSys/commands/getspec.py | 9 | ||||
| -rw-r--r-- | RepSys/commands/getsrpm.py | 33 | ||||
| -rw-r--r-- | RepSys/commands/markrelease.py | 8 | ||||
| -rw-r--r-- | RepSys/commands/patchspec.py | 5 | ||||
| -rw-r--r-- | RepSys/commands/putsrpm.py | 66 | ||||
| -rw-r--r-- | RepSys/commands/rpmlog.py | 36 | ||||
| -rw-r--r-- | RepSys/commands/submit.py | 148 | ||||
| -rw-r--r-- | RepSys/commands/switch.py | 7 | ||||
| -rw-r--r-- | RepSys/commands/sync.py | 4 | ||||
| -rw-r--r-- | RepSys/layout.py | 207 | ||||
| -rw-r--r-- | RepSys/log.py | 286 | ||||
| -rw-r--r-- | RepSys/mirror.py | 101 | ||||
| -rw-r--r-- | RepSys/rpmutil.py | 331 | ||||
| -rw-r--r-- | RepSys/svn.py | 551 | ||||
| -rw-r--r-- | RepSys/util.py | 3 |
24 files changed, 1287 insertions, 596 deletions
diff --git a/RepSys/ConfigParser.py b/RepSys/ConfigParser.py index 4dc3e3c..3b4e213 100644 --- a/RepSys/ConfigParser.py +++ b/RepSys/ConfigParser.py @@ -350,11 +350,12 @@ class Config: self._config = ConfigParser() self._wrapped = {} conffiles = [] - conffiles.append("/etc/repsys.conf") repsys_conf = os.environ.get("REPSYS_CONF") if repsys_conf: conffiles.append(repsys_conf) - conffiles.append(os.path.expanduser("~/.repsys/config")) + else: + conffiles.append("/etc/repsys.conf") + conffiles.append(os.path.expanduser("~/.repsys/config")) for file in conffiles: if os.path.isfile(file): self._config.read(file) diff --git a/RepSys/__init__.py b/RepSys/__init__.py index b303065..2759e8f 100644 --- a/RepSys/__init__.py +++ b/RepSys/__init__.py @@ -11,20 +11,4 @@ del ConfigParser class Error(Exception): pass -class RepSysTree: - """ - This class just hold methods that abstract all the not-so-explicit - rules about the directory structure of a repsys repository. - """ - def fixpath(cls, url): - return re.sub("/+$", "", url) - fixpath = classmethod(fixpath) - - def pkgname(cls, pkgdirurl): - # we must remove trailling slashes in the package path because - # os.path.basename could return "" from URLs ending with "/" - fixedurl = cls.fixpath(pkgdirurl) - return os.path.basename(fixedurl) - pkgname = classmethod(pkgname) - # vim:et:ts=4:sw=4 diff --git a/RepSys/command.py b/RepSys/command.py index 8029e08..f1d61f7 100644 --- a/RepSys/command.py +++ b/RepSys/command.py @@ -1,6 +1,7 @@ #!/usr/bin/python from RepSys import Error, config -import sys, os, urllib +import sys, os +import urlparse import optparse __all__ = ["OptionParser", "do_command", "default_parent"] @@ -39,6 +40,10 @@ def do_command(parse_options_func, main_func): except Error, e: sys.stderr.write("error: %s\n" % str(e)) sys.exit(1) + except KeyboardInterrupt: + sys.stderr.write("interrupted\n") + sys.stderr.flush() + sys.exit(1) def default_parent(url): if url.find("://") == -1: @@ -46,8 +51,9 @@ def default_parent(url): if not default_parent: raise Error, "received a relative url, " \ "but default_parent was not setup" - type, rest = urllib.splittype(default_parent) - url = type+':'+os.path.normpath(rest+'/'+url) + parsed = list(urlparse.urlparse(default_parent)) + parsed[2] = os.path.normpath(parsed[2] + "/" + url) + url = urlparse.urlunparse(parsed) return url # vim:et:ts=4:sw=4 diff --git a/RepSys/commands/authoremail.py b/RepSys/commands/authoremail.py index aee7b58..f5b8b70 100644 --- a/RepSys/commands/authoremail.py +++ b/RepSys/commands/authoremail.py @@ -7,6 +7,9 @@ import getopt HELP = """\ Usage: repsys authoremail [OPTIONS] AUTHOR +Shows the e-mail of an SVN author. It is just a simple interface to access +the [authors] section of repsys.conf. + Options: -h Show this message diff --git a/RepSys/commands/changed.py b/RepSys/commands/changed.py index d3094a8..7d05604 100644 --- a/RepSys/commands/changed.py +++ b/RepSys/commands/changed.py @@ -1,6 +1,7 @@ #!/usr/bin/python from RepSys import Error from RepSys.command import * +from RepSys.layout import package_url from RepSys.rpmutil import check_changed import getopt import sys @@ -8,9 +9,12 @@ import sys HELP = """\ Usage: repsys changed [OPTIONS] URL +Shows if there are pending changes since the last package release. + Options: -a Check all packages in given URL -s Show differences + -M Do not use the mirror (use the main repository) -h Show this message Examples: @@ -25,7 +29,7 @@ def parse_options(): opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = package_url(args[0]) opts.verbose = 1 # Unconfigurable return opts diff --git a/RepSys/commands/ci.py b/RepSys/commands/ci.py index 9ffa3bd..8d373b5 100644 --- a/RepSys/commands/ci.py +++ b/RepSys/commands/ci.py @@ -5,12 +5,16 @@ from RepSys.rpmutil import commit HELP = """\ Usage: repsys ci [TARGET] -Will commit a change. The difference between an ordinary "svn ci" and -"repsys ci" is that it relocates the working copy to the default repository -in case the option "mirror" is set in repsys.conf. +Will commit recent modifications in the package. + +The difference between an ordinary "svn ci" and "repsys ci" is that it +relocates the working copy to the default repository in case the option +"mirror" is set in repsys.conf. Options: -h Show this message + -m MSG Use the MSG as the log message + -F FILE Read log message from FILE Examples: repsys ci @@ -20,6 +24,8 @@ Examples: def parse_options(): parser = OptionParser(help=HELP) parser.add_option("-m", dest="message", default=None) + parser.add_option("-F", dest="logfile", type="string", + default=None) opts, args = parser.parse_args() if len(args): opts.target = args[0] diff --git a/RepSys/commands/co.py b/RepSys/commands/co.py index cadcf56..5349049 100644 --- a/RepSys/commands/co.py +++ b/RepSys/commands/co.py @@ -8,12 +8,24 @@ import sys HELP = """\ Usage: repsys co [OPTIONS] URL [LOCALPATH] +Checkout the package source from the Mandriva repository. + +If the 'mirror' option is enabled, the package is obtained from the mirror +repository. + +You can specify the distro branch to checkout from by using distro/pkgname. + Options: + -d The distribution branch to checkout from + -b The package branch -r REV Revision to checkout - -o Do not use the mirror (use official server) + -M Do not use the mirror (use the main repository) -h Show this message Examples: + repsys co pkgname + repsys co -d 2009.0 pkgname + repsys co 2009.0/pkgame repsys co http://repos/svn/cnc/snapshot/foo repsys co http://repos/svn/cnc/snapshot/foo foo-pkg """ @@ -21,11 +33,13 @@ Examples: def parse_options(): parser = OptionParser(help=HELP) parser.add_option("-r", dest="revision") - parser.add_option("-o", dest="use_mirror", default=True, - action="store_false") + parser.add_option("--distribution", "-d", dest="distro", default=None) + parser.add_option("--branch", "-b", dest="branch", default=None) opts, args = parser.parse_args() if len(args) not in (1, 2): raise Error, "invalid arguments" + # here we don't use package_url in order to notify the user we are + # using the mirror opts.pkgdirurl = args[0] if len(args) == 2: opts.path = args[1] diff --git a/RepSys/commands/create.py b/RepSys/commands/create.py index 56af1ef..ded8abe 100644 --- a/RepSys/commands/create.py +++ b/RepSys/commands/create.py @@ -1,6 +1,7 @@ #!/usr/bin/python from RepSys import Error from RepSys.command import * +from RepSys.layout import package_url from RepSys.rpmutil import create_package import getopt import sys @@ -8,11 +9,14 @@ import sys HELP = """\ Usage: repsys create [OPTIONS] URL +Creates the minimal structure of a package in the repository. + Options: -h Show this message Examples: - repsys create http://repos/svn/cnc/snapshot/newpkg + repsys create newpkg + repsys create svn+ssh://svn.mandriva.com/svn/packages/cooker/newpkg """ def parse_options(): @@ -20,7 +24,7 @@ def parse_options(): opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = package_url(args[0], mirrored=False) opts.verbose = 1 # Unconfigurable return opts diff --git a/RepSys/commands/editlog.py b/RepSys/commands/editlog.py index 367238f..9d1afc5 100644 --- a/RepSys/commands/editlog.py +++ b/RepSys/commands/editlog.py @@ -1,6 +1,7 @@ #!/usr/bin/python from RepSys import Error from RepSys.command import * +from RepSys.layout import package_url from RepSys.svn import SVN import re @@ -24,14 +25,13 @@ def parse_options(): pkgdirurl, revision = "", args[0] else: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(pkgdirurl) + opts.pkgdirurl = package_url(pkgdirurl, mirrored=False) opts.revision = re.compile(r".*?(\d+).*").sub(r"\1", revision) return opts def editlog(pkgdirurl, revision): svn = SVN() - svn.propedit("svn:log", pkgdirurl, revision=SVN.makerev(revision), - revprop=True) + svn.propedit("svn:log", pkgdirurl, rev=revision) def main(): do_command(parse_options, editlog) diff --git a/RepSys/commands/getspec.py b/RepSys/commands/getspec.py index 1079a81..6a8f7ea 100644 --- a/RepSys/commands/getspec.py +++ b/RepSys/commands/getspec.py @@ -1,6 +1,7 @@ #!/usr/bin/python from RepSys import Error from RepSys.command import * +from RepSys.layout import package_url from RepSys.rpmutil import get_spec import getopt import sys @@ -8,12 +9,16 @@ import sys HELP = """\ Usage: repsys getspec [OPTIONS] REPPKGURL +Prints the .spec file of a given package. + Options: -t DIR Use DIR as target for spec file (default is ".") + -M Do not use the mirror (use the main repository) -h Show this message Examples: - repsys getspec http://repos/svn/cnc/snapshot/foo + repsys getspec pkgname + repsys getspec svn+ssh://svn.mandriva.com/svn/packages/cooker/pkgname """ def parse_options(): @@ -22,7 +27,7 @@ def parse_options(): opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = package_url(args[0]) return opts def main(): diff --git a/RepSys/commands/getsrpm.py b/RepSys/commands/getsrpm.py index d76aca7..8cbe1f1 100644 --- a/RepSys/commands/getsrpm.py +++ b/RepSys/commands/getsrpm.py @@ -5,6 +5,7 @@ # from RepSys import Error, config from RepSys.command import * +from RepSys.layout import package_url from RepSys.rpmutil import get_srpm import tempfile import shutil @@ -16,20 +17,26 @@ import os HELP = """\ Usage: repsys getsrpm [OPTIONS] REPPKGURL +Generates the source RPM (.srpm) file of a given package. + Options: - -c Use files in current/ directory (default) - -p Use files in pristine/ directory - -v VER Use files from the version specified by VER (e.g. 2.2.1-2cl) - -r REV Use files from current directory, in revision REV (e.g. 1001) - -t DIR Put SRPM file in directory DIR when done (default is ".") - -P USER Define the RPM packager inforamtion to USER - -s FILE Run script with "FILE TOPDIR SPECFILE" command - -n Rename the package to include the revision number - -l Use subversion log to build rpm %changelog - -T FILE Template to be used to generate the %changelog - -h Show this message + -c Use files in current/ directory (default) + -p Use files in pristine/ directory + -v VER Use files from the version specified by VER (e.g. 2.2.1-2cl) + -r REV Use files from current directory, in revision REV (e.g. 1001) + -t DIR Put SRPM file in directory DIR when done (default is ".") + -P USER Define the RPM packager inforamtion to USER + -s FILE Run script with "FILE TOPDIR SPECFILE" command + -n Rename the package to include the revision number + -l Use subversion log to build rpm %changelog + -T FILE Template to be used to generate the %changelog + -M Do not use the mirror (use the main repository) + -h Show this message + --strict Check if the given revision contains changes in REPPKGURL Examples: + repsys getsrpm python + repsys getsrpm -l python repsys getsrpm http://foo.bar/svn/cnc/snapshot/python repsys getsrpm -p http://foo.bar/svn/cnc/releases/8cl/python repsys getsrpm -r 1001 file:///svn/cnc/snapshot/python @@ -69,11 +76,13 @@ def parse_options(): parser.add_option("-n", dest="revname", action="store_true") parser.add_option("-l", dest="svnlog", action="store_true") parser.add_option("-T", dest="template", type="string", default=None) + parser.add_option("--strict", dest="strict", default=False, + action="store_true") opts, args = parser.parse_args() del opts.__ignore if len(args) != 1: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = package_url(args[0]) opts.verbose = 1 return opts diff --git a/RepSys/commands/markrelease.py b/RepSys/commands/markrelease.py index 440775b..057cf1d 100644 --- a/RepSys/commands/markrelease.py +++ b/RepSys/commands/markrelease.py @@ -9,6 +9,7 @@ # from RepSys import Error from RepSys.command import * +from RepSys.layout import package_url from RepSys.simplerpm import SRPM from RepSys.rpmutil import mark_release from RepSys.util import get_auth @@ -21,6 +22,11 @@ HELP = """\ Usage: repsys markrelease [OPTIONS] REPPKGURL +This subcommand creates a 'tag' for a given revision of a given package. + +The tag will be stored in the directory releases/ inside the package +structure. + Options: -f FILE Try to extract information from given file -r REV Revision which will be used to make the release copy tag @@ -55,7 +61,7 @@ def parse_options(): if len(args) != 1: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = package_url(args[0], mirrored=False) filename = opts.filename appendname = opts.appendname diff --git a/RepSys/commands/patchspec.py b/RepSys/commands/patchspec.py index 155ff4f..9a4881b 100644 --- a/RepSys/commands/patchspec.py +++ b/RepSys/commands/patchspec.py @@ -5,12 +5,15 @@ from RepSys import Error from RepSys.rpmutil import patch_spec from RepSys.command import * +from RepSys.layout import package_url import getopt import sys HELP = """\ Usage: repsys patchspec [OPTIONS] REPPKGURL PATCHFILE +It will try to patch a spec file from a given package url. + Options: -l LOG Use LOG as log message -h Show this message @@ -25,7 +28,7 @@ def parse_options(): opts, args = parser.parse_args() if len(args) != 2: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = package_url(args[0], mirrored=False) opts.patchfile = args[1] return opts diff --git a/RepSys/commands/putsrpm.py b/RepSys/commands/putsrpm.py index 21ad234..5256cba 100644 --- a/RepSys/commands/putsrpm.py +++ b/RepSys/commands/putsrpm.py @@ -1,60 +1,54 @@ #!/usr/bin/python -# -# This program will append a release to the Conectiva Linux package -# repository system. It's meant to be a startup system to include -# pre-packaged SRPMS in the repository, thus, you should not commit -# packages over an ongoing package structure (with changes in current/ -# directory and etc). Also, notice that packages must be included in -# cronological order. -# from RepSys import Error from RepSys.command import * +from RepSys.layout import package_url from RepSys.rpmutil import put_srpm import getopt import sys, os HELP = """\ -*** WARNING --- You probably SHOULD NOT use this program! --- WARNING *** +Usage: repsys putsrpm [OPTIONS] SOURCERPMS -Usage: repsys putsrpm [OPTIONS] REPPKGURL +Will import source RPMs into the SVN repository. + +If the package was already imported, it will add the new files and remove +those not present in the source RPM. Options: - -n Append package name to provided URL - -l LOG Use log when commiting changes + -m LOG Log message used when commiting changes + -t Create version-release tag on releases/ + -b NAME The distribution branch to place it + -d URL The URL of base directory where packages will be placed + -c URL The URL of the base directory where the changelog will be + placed + -s Don't strip the changelog from the spec -h Show this message Examples: - repsys putsrpm file://svn/cnc/snapshot/foo /cnc/d/SRPMS/foo-1.0.src.rpm + repsys putsrpm pkg/SRPMS/pkg-2.0-1.src.rpm + repsys putsrpm -b 2009.1 foo-1.1-1.src.rpm """ def parse_options(): parser = OptionParser(help=HELP) - parser.add_option("-l", dest="log", default="") - parser.add_option("-n", dest="appendname", action="store_true") + parser.add_option("-l", dest="logmsg", default="") + parser.add_option("-t", dest="markrelease", action="store_true", + default=False) + parser.add_option("-s", dest="striplog", action="store_false", + default=True) + parser.add_option("-b", dest="branch", type="string", default=None) + parser.add_option("-d", dest="baseurl", type="string", default=None) + parser.add_option("-c", dest="baseold", type="string", default=None) opts, args = parser.parse_args() - if len(args) != 2: - raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) - opts.srpmfile = args[1] + opts.srpmfiles = args return opts -def put_srpm_cmd(pkgdirurl, srpmfile, appendname=0, log=""): - if os.path.isdir(srpmfile): - dir = srpmfile - for entry in os.listdir(dir): - if entry[-8:] == ".src.rpm": - sys.stderr.write("Putting %s... " % entry) - sys.stderr.flush() - entrypath = os.path.join(dir, entry) - try: - put_srpm(pkgdirurl, entrypath, appendname, log) - sys.stderr.write("done\n") - except Error, e: - sys.stderr.write("error: %s\n" % str(e)) - else: - put_srpm(pkgdirurl, srpmfile, appendname, log) - - +def put_srpm_cmd(srpmfiles, markrelease=False, striplog=True, branch=None, + baseurl=None, baseold=None, logmsg=None): + for path in srpmfiles: + put_srpm(path, markrelease, striplog, branch, baseurl, baseold, + logmsg) + def main(): do_command(parse_options, put_srpm_cmd) diff --git a/RepSys/commands/rpmlog.py b/RepSys/commands/rpmlog.py index 7ea1ac0..11fe36d 100644 --- a/RepSys/commands/rpmlog.py +++ b/RepSys/commands/rpmlog.py @@ -3,23 +3,33 @@ # This program will convert the output of "svn log" to be suitable # for usage in an rpm %changelog session. # -from RepSys import Error +from RepSys import Error, layout from RepSys.command import * -from RepSys.log import svn2rpm +from RepSys.svn import SVN +from RepSys.log import get_changelog, split_spec_changelog +from cStringIO import StringIO import getopt +import os import sys HELP = """\ Usage: repsys rpmlog [OPTIONS] REPPKGDIRURL +Prints the RPM changelog of a given package. + Options: -r REV Collect logs from given revision to revision 0 -n NUM Output only last NUM entries -T FILE %changelog template file to be used + -o Append old package changelog + -p Append changelog found in .spec file + -s Sort changelog entries, even from the old log + -M Do not use the mirror (use the main repository) -h Show this message Examples: - repsys rpmlog https://repos/snapshot/python + repsys rpmlog python + repsys rpmlog http://svn.mandriva.com/svn/packages/cooker/python """ def parse_options(): @@ -27,14 +37,28 @@ def parse_options(): parser.add_option("-r", dest="revision") parser.add_option("-n", dest="size", type="int") parser.add_option("-T", "--template", dest="template", type="string") + parser.add_option("-o", dest="oldlog", default=False, + action="store_true") + parser.add_option("-p", dest="usespec", default=False, + action="store_true") + parser.add_option("-s", dest="sort", default=False, + action="store_true") opts, args = parser.parse_args() if len(args) != 1: raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + opts.pkgdirurl = layout.package_url(args[0]) return opts -def rpmlog(pkgdirurl, revision, size, template): - sys.stdout.write(svn2rpm(pkgdirurl, revision, size, template=template)) +def rpmlog(pkgdirurl, revision, size, template, oldlog, usespec, sort): + another = None + if usespec: + svn = SVN() + specurl = layout.package_spec_url(pkgdirurl) + rawspec = svn.cat(specurl, rev=revision) + spec, another = split_spec_changelog(StringIO(rawspec)) + newlog = get_changelog(pkgdirurl, another=another, rev=revision, + size=size, sort=sort, template=template, oldlog=oldlog) + sys.stdout.writelines(newlog) def main(): do_command(parse_options, rpmlog) diff --git a/RepSys/commands/submit.py b/RepSys/commands/submit.py index 5c95526..88ff596 100644 --- a/RepSys/commands/submit.py +++ b/RepSys/commands/submit.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from RepSys import Error, config +from RepSys import Error, config, layout from RepSys.command import * from RepSys.rpmutil import get_spec, get_submit_info from RepSys.util import get_auth, execcmd, get_helper @@ -7,19 +7,29 @@ import urllib import getopt import sys import re - -#try: -# import NINZ.client -#except ImportError: -# NINZ = None +import subprocess import xmlrpclib HELP = """\ -Usage: repsys submit [OPTIONS] [URL [REVISION]] +Usage: repsys submit [OPTIONS] [URL[@REVISION] ...] Submits the package from URL to the submit host. +The submit host will try to build the package, and upon successful +completion will 'tag' the package and upload it to the official +repositories. + +The package name can refer to an alias to a group of packages defined in +the section submit-groups of the configuration file. + +The status of the submit can visualized at: + +http://kenobi.mandriva.com/bs/output.php + +If no URL and revision are specified, the latest changed revision in the +package working copy of the current directory will be used. + Options: -t TARGET Submit given package URL to given target -l Just list available targets @@ -28,53 +38,101 @@ Options: -s The host in which the package URL will be submitted (defaults to the host in the URL) -h Show this message + --distro The distribution branch where the packages come from --define Defines one variable to be used by the submit scripts in the submit host Examples: repsys submit - repsys submit foo 14800 - repsys submit https://repos/svn/mdv/cooker/foo 14800 - repsys submit -r 14800 https://repos/svn/mdv/cooker/foo + repsys submit foo + repsys submit 2009.1/foo + repsys submit foo@14800 bar baz@11001 + repsys submit https://repos/svn/mdv/cooker/foo repsys submit -l https://repos + repsys submit 2008.1/my-packages@11011 + repsys submit --define section=main/testing -t 2008.1 """ +DEFAULT_TARGET = "Cooker" + def parse_options(): parser = OptionParser(help=HELP) - parser.defaults["revision"] = "" - parser.add_option("-t", dest="target", default="Cooker") - parser.add_option("-l", dest="list", action="store_true") + parser.defaults["revision"] = None + parser.add_option("-t", dest="target", default=None) + parser.add_option("-l", action="callback", callback=list_targets) parser.add_option("-r", dest="revision", type="string", nargs=1) parser.add_option("-s", dest="submithost", type="string", nargs=1, default=None) - parser.add_option("--define", action="append") + parser.add_option("--distro", dest="distro", type="string", + default=None) + parser.add_option("--define", action="append", default=[]) opts, args = parser.parse_args() if not args: - name, rev = get_submit_info(".") - try: - yn = raw_input("Submit '%s', revision %d (y/N)? " % (name, rev)) - except KeyboardInterrupt: - yn = "n" - if yn.lower() in ("y", "yes"): - args = name, str(rev) + name, url, rev = get_submit_info(".") + args = ["%s@%s" % (url, str(rev))] + print "Submitting %s at revision %s" % (name, rev) + print "URL: %s" % url + if opts.revision is not None: + # backwards compatibility with the old -r usage + if len(args) == 1: + args[0] = args[0] + "@" + opts.revision else: - print "Cancelled." - sys.exit(1) - elif len(args) > 2: - raise Error, "invalid arguments" - opts.pkgdirurl = default_parent(args[0]) + raise Error, "can't use -r REV with more than one package name" + del opts.revision if len(args) == 2: - opts.revision = re.compile(r".*?(\d+).*").sub(r"\1", args[1]) - elif len(args) == 1 and opts.revision: - # accepts -r 3123 http://foo/bar - pass - elif not opts.list: - raise Error, "provide -l or a revision number" + # prevent from using the old <name> <rev> syntax + try: + rev = int(args[1]) + except ValueError: + # ok, it is a package name, let it pass + pass + else: + raise Error, "the format <name> <revision> is deprecated, "\ + "use <name>@<revision> instead" + # expand group aliases + expanded = [] + for nameurl in args: + expanded.extend(expand_group(nameurl)) + if expanded != args: + print "Submitting: %s" % " ".join(expanded) + args = expanded + opts.urls = [layout.package_url(nameurl, distro=opts.distro, mirrored=False) + for nameurl in args] + if opts.target is None and opts.distro is None: + target = layout.distro_branch(opts.urls[0]) or DEFAULT_TARGET + print "Implicit target: %s" % target + opts.target = target + del opts.distro return opts -def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None): - #if not NINZ: - # raise Error, "you must have NINZ installed to use this command" +def expand_group(group): + name, rev = layout.split_url_revision(group) + distro = None + if "/" in name: + distro, name = name.rsplit("/", 1) + found = config.get("submit-groups", name) + packages = [group] + if found: + packages = found.split() + if rev: + packages = [("%s@%s" % (package, rev)) + for package in packages] + if distro: + packages = ["%s/%s" % (distro, package) + for package in packages] + return packages + +def list_targets(option, opt, val, parser): + host = config.get("submit", "host") + if host is None: + raise Error, "no submit host defined in repsys.conf" + createsrpm = get_helper("create-srpm") + #TODO make it configurable + command = "ssh %s %s --list" % (host, createsrpm) + execcmd(command, show=True) + sys.exit(0) + +def submit(urls, target, define=[], submithost=None): if submithost is None: submithost = config.get("submit", "host") if submithost is None: @@ -86,13 +144,20 @@ def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None): del type, user, port, path, rest # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory - if list: - raise Error, "unable to list targets from svn+ssh:// URLs" createsrpm = get_helper("create-srpm") - command = "ssh %s %s '%s' -r %s -t %s" % ( - submithost, createsrpm, pkgdirurl, revision, target) - if define: - command += " " + " ".join([ "--define " + x for x in define ]) + args = ["ssh", submithost, createsrpm, "-t", target] + for entry in define: + args.append("--define") + args.append(entry) + if len(urls) == 1: + # be compatible with server-side repsys versions older than 1.6.90 + url, rev = layout.split_url_revision(urls[0]) + args.append(url) + args.append("-r") + args.append(str(rev)) + else: + args.extend(urls) + command = subprocess.list2cmdline(args) status, output = execcmd(command) if status == 0: print "Package submitted!" @@ -100,7 +165,6 @@ def submit(pkgdirurl, revision, target, list=0, define=[], submithost=None): sys.stderr.write(output) sys.exit(status) - def main(): do_command(parse_options, submit) diff --git a/RepSys/commands/switch.py b/RepSys/commands/switch.py index dcbdd17..5cbe2d7 100644 --- a/RepSys/commands/switch.py +++ b/RepSys/commands/switch.py @@ -5,9 +5,10 @@ from RepSys.rpmutil import switch HELP = """\ Usage: repsys switch [URL] -Relocates the working copy to the base location URL. If URL is not -provided, it will use the option default_parent from repsys.conf as -default, or, if the current working copy is already based in +Relocates the working copy to the base location URL. + +If URL is not provided, it will use the option default_parent from +repsys.conf as default, or, if the current working copy is already based in default_parent, it will use the location from the mirror option from repsys.conf. diff --git a/RepSys/commands/sync.py b/RepSys/commands/sync.py index 42ede8d..a51db22 100644 --- a/RepSys/commands/sync.py +++ b/RepSys/commands/sync.py @@ -12,6 +12,8 @@ from the spec file. Options: --dry-run Print results without changing the working copy + --download -d + Try to download the source files not found -h Show this message Examples: @@ -22,6 +24,8 @@ def parse_options(): parser = OptionParser(help=HELP) parser.add_option("--dry-run", dest="dryrun", default=False, action="store_true") + parser.add_option("-d", "--download", dest="download", default=False, + action="store_true") opts, args = parser.parse_args() if len(args): opts.target = args[0] diff --git a/RepSys/layout.py b/RepSys/layout.py new file mode 100644 index 0000000..a4a3846 --- /dev/null +++ b/RepSys/layout.py @@ -0,0 +1,207 @@ +""" Handles repository layout scheme and package URLs.""" + +import os +import urlparse + +from RepSys import Error, config +from RepSys.svn import SVN + +__all__ = ["package_url", "checkout_url", "repository_url", "get_url_revision"] + +def layout_dirs(): + devel_branch = config.get("global", "trunk-dir", "cooker/") + devel_branch = os.path.normpath(devel_branch) + branches_dir = config.get("global", "branches-dir", "updates/") + branches_dir = os.path.normpath(branches_dir) + return devel_branch, branches_dir + +def get_url_revision(url, retrieve=True): + """Get the revision from a given URL + + If the URL contains an explicit revision number (URL@REV), just use it + without even checking if the revision really exists. + + The parameter retrieve defines whether it must ask the SVN server for + the revision number or not when it is not found in the URL. + """ + url, rev = split_url_revision(url) + if rev is None and retrieve: + # if no revspec was found, ask the server + svn = SVN() + rev = svn.revision(url) + return rev + +def unsplit_url_revision(url, rev): + if rev is None: + newurl = url + else: + parsed = list(urlparse.urlparse(url)) + path = os.path.normpath(parsed[2]) + parsed[2] = path + "@" + str(rev) + newurl = urlparse.urlunparse(parsed) + return newurl + +def split_url_revision(url): + """Returns a tuple (url, rev) from an subversion URL with @REV + + If the revision is not present in the URL, rev is None. + """ + parsed = list(urlparse.urlparse(url)) + path = os.path.normpath(parsed[2]) + dirs = path.rsplit("/", 1) + lastname = dirs[-1] + newname = lastname + index = lastname.rfind("@") + rev = None + if index != -1: + newname = lastname[:index] + rawrev = lastname[index+1:] + if rawrev: + try: + rev = int(rawrev) + if rev < 0: + raise ValueError + except ValueError: + raise Error, "invalid revision specification on URL: %s" % url + dirs[-1] = newname + newpath = "/".join(dirs) + parsed[2] = newpath + newurl = urlparse.urlunparse(parsed) + return newurl, rev + +def checkout_url(pkgdirurl, branch=None, version=None, release=None, + releases=False, pristine=False, append_path=None): + """Get the URL of a branch of the package, defaults to current/ + + It tries to preserve revisions in the format @REV. + """ + parsed = list(urlparse.urlparse(pkgdirurl)) + path, rev = split_url_revision(parsed[2]) + if releases: + path = os.path.normpath(path + "/releases") + elif version: + assert release is not None + path = os.path.normpath(path + "/releases/" + version + "/" + release) + elif pristine: + path = os.path.join(path, "pristine") + elif branch: + path = os.path.join(path, "branches", branch) + else: + path = os.path.join(path, "current") + if append_path: + path = os.path.join(path, append_path) + path = unsplit_url_revision(path, rev) + parsed[2] = path + newurl = urlparse.urlunparse(parsed) + return newurl + +def convert_default_parent(url): + """Removes the cooker/ component from the URL""" + parsed = list(urlparse.urlparse(url)) + path = os.path.normpath(parsed[2]) + rest, last = os.path.split(path) + parsed[2] = rest + newurl = urlparse.urlunparse(parsed) + return newurl + +def remove_current(pkgdirurl): + parsed = list(urlparse.urlparse(pkgdirurl)) + path = os.path.normpath(parsed[2]) + rest, last = os.path.split(path) + if last == "current": + # FIXME this way we will not allow packages to be named "current" + path = rest + parsed[2] = path + newurl = urlparse.urlunparse(parsed) + return newurl + +def repository_url(mirrored=False): + url = None + if mirrored and config.get("global", "use-mirror"): + url = config.get("global", "mirror") + if url is None: + url = config.get("global", "repository") + if not url: + # compatibility with the default_parent configuration option + default_parent = config.get("global", "default_parent") + if default_parent is None: + raise Error, "you need to set the 'repository' " \ + "configuration option on repsys.conf" + url = convert_default_parent(default_parent) + return url + +def package_url(name_or_url, version=None, release=None, distro=None, + mirrored=True): + """Returns a tuple with the absolute package URL and its name + + @name_or_url: name, relative path, or URL of the package. In case it is + a URL, the URL will just be 'normalized'. + @version: the version to be fetched from releases/ (requires release) + @release: the release number to be fetched from releases/$version/ + @distro: the name of the repository branch inside updates/ + @mirrored: return an URL based on the mirror repository, if enabled + """ + from RepSys import mirror + if "://" in name_or_url: + pkgdirurl = mirror.normalize_path(name_or_url) + pkgdirurl = remove_current(pkgdirurl) + if mirror.using_on(pkgdirurl) and not mirrored: + pkgdirurl = mirror.relocate_path(mirror.mirror_url(), + repository_url(), pkgdirurl) + else: + name = name_or_url + devel_branch, branches_dir = layout_dirs() + if distro or "/" in name: + default_branch = branches_dir + if distro: + default_branch = os.path.join(default_branch, distro) + else: + default_branch = devel_branch # cooker + path = os.path.join(default_branch, name) + parsed = list(urlparse.urlparse(repository_url(mirrored=mirrored))) + parsed[2] = os.path.join(parsed[2], path) + pkgdirurl = urlparse.urlunparse(parsed) + return pkgdirurl + +def package_name(pkgdirurl): + """Returns the package name from a package URL + + It takes care of revision numbers""" + parsed = urlparse.urlparse(pkgdirurl) + path, rev = split_url_revision(parsed[2]) + rest, name = os.path.split(path) + return name + +def package_spec_url(pkgdirurl, *args, **kwargs): + """Returns the URL of the specfile of a given package URL + + The parameters are the same used by checkout_url, except append_path. + """ + kwargs["append_path"] = "SPECS/" + package_name(pkgdirurl) + ".spec" + specurl = checkout_url(pkgdirurl, *args, **kwargs) + return specurl + +def distro_branch(pkgdirurl): + """Tries to guess the distro branch name from a package URL""" + from RepSys.mirror import same_base + found = None + repo = repository_url() + if same_base(repo, pkgdirurl): + devel_branch, branches_dir = layout_dirs() + repo_path = urlparse.urlparse(repo)[2] + devel_path = os.path.join(repo_path, devel_branch) + branches_path = os.path.join(repo_path, branches_dir) + parsed = urlparse.urlparse(pkgdirurl) + path = os.path.normpath(parsed[2]) + if path.startswith(devel_path): + # devel_branch must be before branches_dir in order to allow + # devel_branch to be inside branches_dir, as in /branches/cooker + _, found = os.path.split(devel_branch) + elif path.startswith(branches_path): + comps = path.split("/") + if branches_path == "/": + found = comps[1] + elif len(comps) >= 2: # must be at least branch/pkgname + found = comps[branches_path.count("/")+1] + return found + diff --git a/RepSys/log.py b/RepSys/log.py index 2c69693..a1d1944 100644 --- a/RepSys/log.py +++ b/RepSys/log.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from RepSys import Error, config, RepSysTree +from RepSys import Error, config, layout from RepSys.svn import SVN from RepSys.util import execcmd @@ -8,30 +8,50 @@ try: except ImportError: raise Error, "repsys requires the package python-cheetah" +from cStringIO import StringIO + import sys import os import re import time import locale -import codecs import glob import tempfile import shutil + +locale.setlocale(locale.LC_ALL, "C") + default_template = """ +#if not $releases_by_author[-1].visible + ## Hide the first release that contains no changes. It must be a + ## reimported package and the log gathered from misc/ already should + ## contain a correct entry for the version-release: + #set $releases_by_author = $releases_by_author[:-1] +#end if #for $rel in $releases_by_author * $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release - ## - #if not $rel.released - (not released yet) ++ Revision: $rel.revision +## #if not $rel.released +##+ Status: not released +## #end if + #if not $rel.visible ++ rebuild (emptylog) #end if #for $rev in $rel.release_revisions #for $line in $rev.lines - $line +$line #end for #end for #for $author in $rel.authors + #if not $author.visible + #continue + #end if + ##alternatively, one could use: + ###if $author.email == "root" + ## #continue + ###end if + $author.name <$author.email> #for $rev in $author.revisions #for $line in $rev.lines @@ -50,14 +70,13 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None): Is here where things should be changed if "automatic release increasing" will be used. """ - svn = SVN() from RepSys.rpmutil import rpm_macros_defs - tmpdir = tempfile.mktemp() + svn = SVN() pkgcurrenturl = os.path.join(pkgdirurl, "current") specurl = os.path.join(pkgcurrenturl, "SPECS") if exported is None: tmpdir = tempfile.mktemp() - svn.export(specurl, tmpdir, revision=SVN.makerev(rev)) + svn.export(specurl, tmpdir, rev=rev) else: tmpdir = os.path.join(exported, "SPECS") try: @@ -89,8 +108,7 @@ def getrelease(pkgdirurl, rev=None, macros=[], exported=None): if exported is None and os.path.isdir(tmpdir): shutil.rmtree(tmpdir) - -class ChangelogRevision: +class _Revision: lines = [] date = None raw_date = None @@ -103,11 +121,12 @@ class ChangelogRevision: def __repr__(self): lines = repr(self.lines)[:30] + "...]" - line = "<ChangelogRevision %d author=%r date=%r lines=%s>" % \ + line = "<_Revision %d author=%r date=%r lines=%s>" % \ (self.revision, self.author, self.date, lines) return line -class ChangelogRelease(ChangelogRevision): + +class _Release(_Revision): version = None release = None revisions = [] @@ -116,11 +135,11 @@ class ChangelogRelease(ChangelogRevision): visible = False def __init__(self, **kwargs): - ChangelogRevision.__init__(self, **kwargs) self.revisions = [] + _Revision.__init__(self, **kwargs) def __repr__(self): - line = "<ChangelogRelease v=%s r=%s revs=%r>" % \ + line = "<_Release v=%s r=%s revs=%r>" % \ (self.version, self.release, self.revisions) return line @@ -153,59 +172,62 @@ def format_lines(lines): return entrylines -class ChangelogByAuthor: +class _Author: name = None email = None revisions = None + visible = False def group_releases_by_author(releases): allauthors = [] grouped = [] for release in releases: + + # group revisions of the release by author authors = {} latest = None for revision in release.revisions: authors.setdefault(revision.author, []).append(revision) - # all the mess below is to sort by author and by revision number + # create _Authors and sort them by their latest revisions decorated = [] for authorname, revs in authors.iteritems(): - author = ChangelogByAuthor() + author = _Author() author.name = revs[0].author_name author.email = revs[0].author_email - revdeco = [(r.revision, r) for r in revs] - revdeco.sort(reverse=1) - author.revisions = [t[1] for t in revdeco] + author.revisions = revs + # #41117: mark those authors without visible messages + author.visible = bool(sum(len(rev.lines) for rev in revs)) revlatest = author.revisions[0] - # keep the latest revision even for silented authors (below) + # keep the latest revision even for completely invisible + # authors (below) if latest is None or revlatest.revision > latest.revision: latest = revlatest - count = sum(len(rev.lines) for rev in author.revisions) - if count == 0: - # skipping author with only silented lines + if not author.visible: + # only sort those visible authors, invisible ones are used + # only in "latest" continue - decorated.append((revdeco[0][0], author)) + decorated.append((revlatest.revision, author)) + decorated.sort(reverse=1) - if not decorated: - # skipping release with only authors with silented lines - continue + if release.visible: + release.authors = [t[1] for t in decorated] + firstrel, release.authors = release.authors[0], release.authors[1:] + release.author_name = firstrel.name + release.author_email = firstrel.email + release.release_revisions = firstrel.revisions + else: + # we don't care about other possible authors in completely + # invisible releases + firstrev = release.revisions[0] + release.author_name = firstrev.author_name + release.author_email = firstrev.author_email + release.raw_date = firstrev.raw_date + release.date = firstrev.date - decorated.sort(reverse=1) - release.authors = [t[1] for t in decorated] - # the difference between a released and a not released _Release is - # the way the release numbers is obtained. So, when this is a - # released, we already have it, but if we don't, we should get de - # version/release string using getrelease and then get the first - first, release.authors = release.authors[0], release.authors[1:] - release.author_name = first.name - release.author_email = first.email - release.release_revisions = first.revisions - - #release.date = first.revisions[0].date release.date = latest.date release.raw_date = latest.raw_date - #release.revision = first.revisions[0].revision release.revision = latest.revision grouped.append(release) @@ -217,7 +239,7 @@ def group_revisions_by_author(currentlog): revisions = [] last_author = None for entry in currentlog: - revision = ChangelogRevision() + revision = _Revision() revision.lines = format_lines(entry.lines) revision.raw_date = entry.date revision.date = parse_raw_date(entry.date) @@ -225,7 +247,7 @@ def group_revisions_by_author(currentlog): if entry.author == last_author: revisions[-1].revisions.append(revision) else: - author = ChangelogByAuthor() + author = _Author() author.name, author.email = get_author_name(entry.author) author.revisions = [revision] revisions.append(author) @@ -269,7 +291,7 @@ def filter_log_lines(lines): def make_release(author=None, revision=None, date=None, lines=None, entries=[], released=True, version=None, release=None): - rel = ChangelogRelease() + rel = _Release() rel.author = author if author: rel.author_name, rel.author_email = get_author_name(author) @@ -282,11 +304,11 @@ def make_release(author=None, revision=None, date=None, lines=None, rel.visible = False for entry in entries: lines = filter_log_lines(entry.lines) - if lines: - rel.visible = True - revision = ChangelogRevision() + revision = _Revision() revision.revision = entry.revision revision.lines = format_lines(lines) + if revision.lines: + rel.visible = True revision.date = parse_raw_date(entry.date) revision.raw_date = entry.date revision.author = entry.author @@ -297,7 +319,8 @@ def make_release(author=None, revision=None, date=None, lines=None, def dump_file(releases, currentlog=None, template=None): - templpath = template or config.get("template", "path", None) + templpath = template or config.get("template", "path", + "/usr/share/repsys/default.chlog") params = {} if templpath is None or not os.path.exists(templpath): params["source"] = default_template @@ -311,12 +334,7 @@ def dump_file(releases, currentlog=None, template=None): "releases" : releases, "revisions_by_author": revisions_author}] t = Template(**params) - chlog = t.respond() - try: - chlog = chlog.decode("utf8") - except UnicodeError: - pass - return chlog + return t.respond() class InvalidEntryError(Exception): @@ -379,18 +397,14 @@ def parse_markrelease_log(relentry): def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, template=None, macros=[], exported=None): - size = size or 0 concat = config.get("log", "concat", "").split() revoffset = get_revision_offset() svn = SVN() - pkgreleasesurl = os.path.join(pkgdirurl, "releases") - pkgcurrenturl = os.path.join(pkgdirurl, "current") - releaseslog = list(svn.log(pkgreleasesurl, - strict_node_history=False, noerror=1)) or [] - currentlog = list(svn.log(pkgcurrenturl, - strict_node_history=False, - revision_start=SVN.makerev(rev), - revision_end=SVN.makerev(revoffset), limit=size)) + pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True) + pkgcurrenturl = layout.checkout_url(pkgdirurl) + releaseslog = svn.log(pkgreleasesurl, noerror=1) + currentlog = svn.log(pkgcurrenturl, limit=size, start=rev, + end=revoffset) # sort releases by copyfrom-revision, so that markreleases for same # revisions won't look empty @@ -454,40 +468,74 @@ def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, data = dump_file(releases[::-1], currentlog=currentlog, template=template) return data - - -def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, - submit=False, template=None, macros=[], exported=None): - newlines = [] +def _split_changelog(stream): + current = None + count = 0 + def finish(entry): + lines = entry[2] + # strip newlines at the end + for i in xrange(len(lines)-1, -1, -1): + if lines[i] != "\n": + break + del lines[i] + return entry + for line in stream: + if line.startswith("*"): + if current: + yield finish(current) + fields = line.split() + rawdate = " ".join(fields[:5]) + try: + date = time.strptime(rawdate, "* %a %b %d %Y") + except ValueError, e: + raise Error, "failed to parse spec changelog: %s" % e + curlines = [line] + current = (date, count, curlines) + # count used to ensure stable sorting when changelog entries + # have the same date, otherwise it would also compare the + # changelog lines + count -= 1 + elif current: + curlines.append(line) + else: + pass # not good, but ignore + if current: + yield finish(current) + +def sort_changelog(stream): + entries = _split_changelog(stream) + log = StringIO() + for time, count, elines in sorted(entries, reverse=True): + log.writelines(elines) + log.write("\n") + return log + +def split_spec_changelog(stream): + chlog = StringIO() + spec = StringIO() found = 0 - - encoding = locale.getpreferredencoding() - - def open(name, mode="r"): - return codecs.open(name, mode, encoding, errors="replace") - - # Strip old changelogs - for line in open(specfile): + for line in stream: if line.startswith("%changelog"): found = 1 elif not found: - newlines.append(line) + spec.write(line) + elif found: + chlog.write(line) elif line.startswith("%"): found = 0 - newlines.append(line) - - # Create new changelog - newlines.append("\n\n%changelog\n") - newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, - template=template, macros=macros, exported=exported)) + spec.write(line) + spec.seek(0) + chlog.seek(0) + return spec, chlog - # Merge old changelog, if available +def get_old_log(pkgdirurl): + chlog = StringIO() oldurl = config.get("log", "oldurl") if oldurl: svn = SVN() tmpdir = tempfile.mktemp() try: - pkgname = RepSysTree.pkgname(pkgdirurl) + pkgname = layout.package_name(pkgdirurl) pkgoldurl = os.path.join(oldurl, pkgname) try: # we're using HEAD here because fixes in misc/ (oldurl) may @@ -499,20 +547,74 @@ def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, logfile = os.path.join(tmpdir, "log") if os.path.isfile(logfile): file = open(logfile) - newlines.append("\n") + chlog.write("\n") # TODO needed? log = file.read() log = escape_macros(log) - newlines.append(log) + chlog.write(log) file.close() finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) + chlog.seek(0) + return chlog - # Write new specfile - file = open(specfile, "w") - file.write("".join(newlines)) - file.close() +def get_changelog(pkgdirurl, another=None, svn=True, rev=None, size=None, + submit=False, sort=False, template=None, macros=[], exported=None, + oldlog=False): + """Generates the changelog for a given package URL + + @another: a stream with the contents of a changelog to be merged with + the one generated + @svn: enable changelog from svn + @rev: generate the changelog with the changes up to the given + revision + @size: the number of revisions to be used (as in svn log --limit) + @submit: defines whether the latest unreleased log entries should have + the version parsed from the spec file + @sort: should changelog entries be reparsed and sorted after appending + the oldlog? + @template: the path to the cheetah template used to generate the + changelog from svn + @macros: a list of tuples containing macros to be defined when + parsing the version in the changelog + @exported: the path of a directory containing an already existing + checkout of the package, so that the spec file can be + parsed from there + @oldlog: if set it will try to append the old changelog file defined + in oldurl in repsys.conf + """ + newlog = StringIO() + if svn: + rawsvnlog = svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit, + template=template, macros=macros, exported=exported) + newlog.write(rawsvnlog) + if another: + newlog.writelines(another) + if oldlog: + newlog.writelines(get_old_log(pkgdirurl)) + if sort: + newlog.seek(0) + newlog = sort_changelog(newlog) + newlog.seek(0) + return newlog +def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, + submit=False, sort=False, template=None, macros=[], exported=None): + fi = open(specfile) + spec, oldchlog = split_spec_changelog(fi) + fi.close() + another = None + if config.getbool("log", "merge-spec", False): + another = oldchlog + sort = sort or config.getbool("log", "sort", False) + chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size, + submit=submit, sort=sort, template=template, macros=macros, + exported=exported, oldlog=True) + fo = open(specfile, "w") + fo.writelines(spec) + fo.write("\n\n%changelog\n") + fo.writelines(chlog) + fo.close() if __name__ == "__main__": l = svn2rpm(sys.argv[1]) diff --git a/RepSys/mirror.py b/RepSys/mirror.py index 1e9e9c2..f0d2316 100644 --- a/RepSys/mirror.py +++ b/RepSys/mirror.py @@ -1,87 +1,104 @@ +import sys import os import urlparse +import urllib -from RepSys import Error, config +from RepSys import Error, config, layout from RepSys.svn import SVN -def _normdirurl(url): +def mirror_url(): + mirror = config.get("global", "mirror") + return mirror + +def normalize_path(url): """normalize url for relocate_path needs""" parsed = urlparse.urlparse(url) - path = os.path.normpath(parsed.path) - newurl = urlparse.urlunparse((parsed.scheme, parsed.netloc, path, - parsed.params, parsed.query, parsed.fragment)) + path = os.path.normpath(parsed[2]) + newurl = urlparse.urlunparse((parsed[0], parsed[1], path, + parsed[3], parsed[4], parsed[5])) return newurl def _joinurl(url, relpath): parsed = urlparse.urlparse(url) - newpath = os.path.join(parsed.path, relpath) - newurl = urlparse.urlunparse((parsed.scheme, parsed.netloc, newpath, - parsed.params, parsed.query, parsed.fragment)) + newpath = os.path.join(parsed[2], relpath) + newurl = urlparse.urlunparse((parsed[0], parsed[1], newpath, + parsed[3], parsed[4], parsed[5])) + return newurl + + +def strip_username(url): + parsed = list(urlparse.urlparse(url)) + _, parsed[1] = urllib.splituser(parsed[1]) + newurl = urlparse.urlunparse(parsed) return newurl def same_base(parent, url): """returns true if parent is parent of url""" - parent = _normdirurl(parent) - url = _normdirurl(url) - #FIXME handle paths with/without username/password + parent = normalize_path(parent) + url = normalize_path(url) + url = strip_username(url) return url.startswith(parent) def relocate_path(oldparent, newparent, url): - oldparent = _normdirurl(oldparent) - newparent = _normdirurl(newparent) - url = _normdirurl(url) + oldparent = normalize_path(oldparent) + newparent = normalize_path(newparent) + url = normalize_path(url) subpath = url[len(oldparent)+1:] newurl = _joinurl(newparent, subpath) # subpath usually gets / at begining return newurl def enabled(wcurl=None): - mirror = config.get("global", "mirror") - default_parent = config.get("global", "default_parent") + mirror = mirror_url() + repository = layout.repository_url() enabled = False - if mirror and default_parent: + if mirror and repository: enabled = True - if wcurl and (not same_base(mirror, wcurl)): + if wcurl and not same_base(mirror, wcurl): enabled = False return enabled +def using_on(url): + """returnes True if the URL points to the mirror repository""" + mirror = mirror_url() + if mirror: + using = same_base(mirror, url) + else: + using = False + return using + +def info(url, stream=sys.stderr): + if using_on(url): + stream.write("using mirror\n") + def mirror_relocate(oldparent, newparent, url, wcpath): svn = SVN() newurl = relocate_path(oldparent, newparent, url) - # note that svn.relocate requires paths without trailling slashes, - # http://foo/svn/bar/baz/ will fail - svn.relocate(url, newurl, wcpath) + svn.switch(newurl, url, path=wcpath, relocate=True) return newurl def switchto_parent(svn, url, path): """Relocates the working copy to default_parent""" - mirror = config.get("global", "mirror") - default_parent = config.get("global", "default_parent") - newurl = mirror_relocate(mirror, default_parent, url, path) + newurl = mirror_relocate(mirror_url(), layout.repository_url(), url, path) return newurl -def switchto_mirror(svn, url, path): - mirror = config.get("global", "mirror") - default_parent = config.get("global", "default_parent") - newurl = mirror_relocate(default_parent, mirror, url, path) +def switchto_parent_url(url): + newurl = relocate_path(mirror_url(), layout.repository_url(), url) return newurl -def checkout_url(url): - mirror = config.get("global", "mirror") - default_parent = config.get("global", "default_parent") - if mirror is not None and default_parent is not None: - return relocate_path(default_parent, mirror, url) - return url +def switchto_mirror(svn, url, path): + newurl = mirror_relocate(layout.repository_url(), mirror_url(), url, path) + return newurl def autoswitch(svn, wcpath, wcurl, newbaseurl=None): """Switches between mirror, default_parent, or newbaseurl""" nobase = False - mirror = config.get("global", "mirror") - default_parent = config.get("global", "default_parent") - current = default_parent - if default_parent is None: - raise Error, "the option default_parent from repsys.conf is "\ + mirror = mirror_url() + repository = layout.repository_url() + current = repository + if repository is None: + raise Error, "the option repository from repsys.conf is "\ "required" - indefault = same_base(default_parent, wcurl) + indefault = same_base(repository, wcurl) if not newbaseurl: if not mirror: raise Error, "an URL is needed when the option mirror "\ @@ -90,7 +107,7 @@ def autoswitch(svn, wcpath, wcurl, newbaseurl=None): chosen = mirror elif same_base(mirror, wcurl): current = mirror - chosen = default_parent + chosen = repository else: nobase = True else: @@ -103,7 +120,7 @@ def autoswitch(svn, wcpath, wcurl, newbaseurl=None): chosen = newbaseurl if nobase: raise Error, "the URL of this working copy is not based in "\ - "default_parent nor mirror URLs" + "repository nor mirror URLs" assert current != chosen newurl = mirror_relocate(current, chosen, wcurl, wcpath) return newurl diff --git a/RepSys/rpmutil.py b/RepSys/rpmutil.py index 4cbba62..0ca6bdb 100644 --- a/RepSys/rpmutil.py +++ b/RepSys/rpmutil.py @@ -1,15 +1,15 @@ #!/usr/bin/python -from RepSys import Error, config, RepSysTree -from RepSys import mirror +from RepSys import Error, config +from RepSys import mirror, layout, log from RepSys.svn import SVN from RepSys.simplerpm import SRPM -from RepSys.log import specfile_svn2rpm from RepSys.util import execcmd -import pysvn from RepSys.command import default_parent import rpm +import urlparse import tempfile import shutil +import string import glob import sys import os @@ -18,8 +18,8 @@ def get_spec(pkgdirurl, targetdir=".", submit=False): svn = SVN() tmpdir = tempfile.mktemp() try: - geturl = "/".join([pkgdirurl, "current", "SPECS"]) - svn.export("%s" % geturl, tmpdir) + geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") + svn.export("'%s'" % geturl, tmpdir) speclist = glob.glob(os.path.join(tmpdir, "*.spec")) if not speclist: raise Error, "no spec files found" @@ -34,6 +34,23 @@ def rpm_macros_defs(macros): args = " ".join(defs) return args +#FIXME move it to another module +def rev_touched_url(url, rev): + svn = SVN() + info = svn.info2(url) + if info is None: + raise Error, "can't fetch svn info about the URL: %s" % url + root = info["Repository Root"] + urlpath = url[len(root):] + touched = False + entries = svn.log(root, start=rev, limit=1) + entry = entries[0] + for change in entry.changed: + path = change.get("path") + if path and path.startswith(urlpath): + touched = True + return touched + def get_srpm(pkgdirurl, mode = "current", targetdirs = None, @@ -47,7 +64,8 @@ def get_srpm(pkgdirurl, submit = False, template = None, macros = [], - verbose = 0): + verbose = 0, + strict = False): svn = SVN() tmpdir = tempfile.mktemp() topdir = "--define '_topdir %s'" % tmpdir @@ -57,17 +75,26 @@ def get_srpm(pkgdirurl, specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS") srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS") patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES") + try: if mode == "version": - geturl = os.path.join(pkgdirurl, "releases", - version, release) + geturl = layout.checkout_url(pkgdirurl, version=version, + release=release) elif mode == "pristine": - geturl = os.path.join(pkgdirurl, "pristine") + geturl = layout.checkout_url(pkgdirurl, pristine=True) elif mode == "current" or mode == "revision": - geturl = os.path.join(pkgdirurl, "current") + #FIXME we should handle revisions specified using @REV + geturl = layout.checkout_url(pkgdirurl) else: raise Error, "unsupported get_srpm mode: %s" % mode - exportedrev = svn.export(geturl, tmpdir, revision=SVN.makerev(revision)) + strict = strict or config.getbool("submit", "strict-revision", False) + if strict and not rev_touched_url(geturl, revision): + #FIXME would be nice to have the revision number even when + # revision is None + raise Error, "the revision %s does not change anything "\ + "inside %s" % (revision or "HEAD", geturl) + mirror.info(geturl) + svn.export(geturl, tmpdir, rev=revision) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") @@ -77,7 +104,7 @@ def get_srpm(pkgdirurl, spec = speclist[0] if svnlog: submit = not not revision - specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, + log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir) for script in scripts: #FIXME revision can be "None" @@ -89,39 +116,46 @@ def get_srpm(pkgdirurl, packager = " --define 'packager %s'" % packager defs = rpm_macros_defs(macros) - execcmd("rpm -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % - (topdir, builddir, rpmdir, sourcedir, specdir, - srcrpmdir, patchdir, packager, spec, defs)) - - if revision: - #FIXME duplicate glob line - srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0] - srpminfo = SRPM(srpm) - release = srpminfo.release - srpmbase = os.path.basename(srpm) - os.rename(srpm, "%s/@%s:%s" % (srpmsdir, exportedrev.number, srpmbase)) - srpm = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))[0] + sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") + execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % + (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir, + srcrpmdir, patchdir, packager, spec, defs)) + + # copy the generated SRPMs to their target locations + targetsrpms = [] + urlrev = None + if revname: + urlrev = revision or layout.get_url_revision(geturl) if not targetdirs: targetdirs = (".",) - targetsrpms = [] - for targetdir in targetdirs: - targetsrpm = os.path.join(os.path.realpath(targetdir), - os.path.basename(srpm)) - targetsrpms.append(targetsrpm) - if verbose: - sys.stderr.write("Wrote: %s\n" % targetsrpm) - execcmd("cp -f", srpm, targetdir) - os.unlink(srpm) + srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm")) + if not srpms: + # something fishy happened + raise Error, "no SRPMS were found at %s" % srpmsdir + for srpm in srpms: + name = os.path.basename(srpm) + if revname: + name = "@%s:%s" % (urlrev, name) + for targetdir in targetdirs: + newpath = os.path.join(targetdir, name) + targetsrpms.append(newpath) + if os.path.exists(newpath): + # should we warn? + os.unlink(newpath) + shutil.copy(srpm, newpath) + if verbose: + sys.stderr.write("Wrote: %s\n" % newpath) return targetsrpms finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) def patch_spec(pkgdirurl, patchfile, log=""): + #FIXME use get_spec svn = SVN() tmpdir = tempfile.mktemp() try: - geturl = "/".join([pkgdirurl, "current", "SPECS"]) + geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") svn.checkout(geturl, tmpdir) speclist = glob.glob(os.path.join(tmpdir, "*.spec")) if not speclist: @@ -131,39 +165,46 @@ def patch_spec(pkgdirurl, patchfile, log=""): if status != 0: raise Error, "can't apply patch:\n%s\n" % output else: - svn.checkin(tmpdir, log=log) + svn.commit(tmpdir, log="") finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) -def put_srpm(pkgdirurl, srpmfile, appendname=0, log=""): - srpm = SRPM(srpmfile) - if appendname: - pkgdirurl = "/".join([pkgdirurl, srpm.name]) +def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, + baseurl=None, baseold=None, logmsg=None): svn = SVN() + srpm = SRPM(srpmfile) tmpdir = tempfile.mktemp() + if baseurl: + pkgurl = mirror._joinurl(baseurl, srpm.name) + else: + pkgurl = layout.package_url(srpm.name, distro=branch, + mirrored=False) + print "Importing package to %s" % pkgurl try: if srpm.epoch: version = "%s:%s" % (srpm.epoch, srpm.version) else: version = srpm.version - versionurl = "/".join([pkgdirurl, "releases", version]) + versionurl = "/".join([pkgurl, "releases", version]) releaseurl = "/".join([versionurl, srpm.release]) - ret = svn.mkdir(pkgdirurl, "Created package directory", noerror=1) - if ret: - svn.checkout(pkgdirurl, tmpdir) + currenturl = os.path.join(tmpdir, "current") + #FIXME when pre-commit hook fails, there's no clear way to know + # what happened + ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory") + if ret or not svn.ls(currenturl, noerror=1): + svn.checkout(pkgurl, tmpdir) svn.mkdir(os.path.join(tmpdir, "releases")) - svn.mkdir(os.path.join(tmpdir, "releases", version)) - svn.mkdir(os.path.join(tmpdir, "current")) - svn.mkdir(os.path.join(tmpdir, "current", "SPECS")) - svn.mkdir(os.path.join(tmpdir, "current", "SOURCES")) + svn.mkdir(currenturl) + svn.mkdir(os.path.join(currenturl, "SPECS")) + svn.mkdir(os.path.join(currenturl, "SOURCES")) #svn.commit(tmpdir,log="Created package structure.") version_exists = 1 currentdir = os.path.join(tmpdir, "current") else: - if svn.exists(releaseurl): + if svn.ls(releaseurl, noerror=1): raise Error, "release already exists" - svn.checkout("/".join([pkgdirurl, "current"]), tmpdir) + svn.checkout("/".join([pkgurl, "current"]), tmpdir) svn.mkdir(versionurl, noerror=1, log="Created directory for version %s." % version) currentdir = tmpdir @@ -217,28 +258,56 @@ def put_srpm(pkgdirurl, srpmfile, appendname=0, log=""): if os.path.isdir(unpackdir): shutil.rmtree(unpackdir) - svn.checkin(tmpdir, log=log) + if striplog: + specs = glob.glob(os.path.join(specsdir, "*.spec")) + if not specs: + raise Error, "no spec file fount on %s" % specsdir + specpath = specs[0] + fspec = open(specpath) + spec, chlog = log.split_spec_changelog(fspec) + chlog.seek(0) + fspec.close() + oldurl = baseold or config.get("log", "oldurl") + pkgoldurl = mirror._joinurl(oldurl, srpm.name) + svn.mkdir(pkgoldurl, noerror=1, + log="created old log directory for %s" % srpm.name) + logtmp = tempfile.mktemp() + try: + svn.checkout(pkgoldurl, logtmp) + miscpath = os.path.join(logtmp, "log") + fmisc = open(miscpath, "w+") + fmisc.writelines(chlog) + fmisc.close() + svn.add(miscpath) + svn.commit(logtmp, + log="imported old log for %s" % srpm.name) + finally: + if os.path.isdir(logtmp): + shutil.rmtree(logtmp) + svn.commit(tmpdir, + log=logmsg or ("imported package %s" % srpm.name)) finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) # Do revision and pristine tag copies - pristineurl = os.path.join(pkgdirurl, "pristine") + pristineurl = layout.checkout_url(pkgurl, pristine=True) svn.remove(pristineurl, noerror=1, log="Removing previous pristine/ directory.") - currenturl = os.path.join(pkgdirurl, "current") + currenturl = layout.checkout_url(pkgurl) svn.copy(currenturl, pristineurl, log="Copying release %s-%s to pristine/ directory." % (version, srpm.release)) - svn.copy(currenturl, releaseurl, - log="Copying release %s-%s to releases/ directory." % - (version, srpm.release)) + if markrelease: + svn.copy(currenturl, releaseurl, + log="Copying release %s-%s to releases/ directory." % + (version, srpm.release)) def create_package(pkgdirurl, log="", verbose=0): svn = SVN() tmpdir = tempfile.mktemp() try: - basename = RepSysTree.pkgname(pkgdirurl) + basename = layout.package_name(pkgdirurl) if verbose: print "Creating package directory...", sys.stdout.flush() @@ -257,7 +326,7 @@ def create_package(pkgdirurl, log="", verbose=0): if verbose: print "done" print "Committing...", - svn.checkin(tmpdir, + svn.commit(tmpdir, log="Created package structure for '%s'." % basename) print "done" finally: @@ -280,21 +349,21 @@ def mark_release(pkgdirurl, version, release, revision): releasesurl = "/".join([pkgdirurl, "releases"]) versionurl = "/".join([releasesurl, version]) releaseurl = "/".join([versionurl, release]) - if svn.exists(releaseurl, noerror=1): + if svn.ls(releaseurl, noerror=1): raise Error, "release already exists" svn.mkdir(releasesurl, noerror=1, log="Created releases directory.") svn.mkdir(versionurl, noerror=1, log="Created directory for version %s." % version) - pristineurl = os.path.join(pkgdirurl, "pristine") + pristineurl = layout.checkout_url(pkgdirurl, pristine=True) svn.remove(pristineurl, noerror=1, log="Removing previous pristine/ directory.") - currenturl = os.path.join(pkgdirurl, "current") + currenturl = layout.checkout_url(pkgdirurl) svn.copy(currenturl, pristineurl, log="Copying release %s-%s to pristine/ directory." % (version, release)) markreleaselog = create_markrelease_log(version, release, revision) - svn.copy(currenturl, releaseurl, revision=revision, + svn.copy(currenturl, releaseurl, rev=revision, log=markreleaselog) def check_changed(pkgdirurl, all=0, show=0, verbose=0): @@ -305,29 +374,30 @@ def check_changed(pkgdirurl, all=0, show=0, verbose=0): if verbose: print "Getting list of packages...", sys.stdout.flush() - packages = [x['name'] for x in svn.ls(baseurl)] + packages = [x[:-1] for x in svn.ls(baseurl)] if verbose: print "done" if not packages: raise Error, "couldn't get list of packages" else: - packages = [pkgdirurl] + baseurl, basename = os.path.split(pkgdirurl) + packages = [basename] clean = [] changed = [] nopristine = [] nocurrent = [] - for pkgdirurl in packages: - package = os.path.basename(pkgdirurl) - current = os.path.join(pkgdirurl, "current") - pristine = os.path.join(pkgdirurl, "pristine") + for package in packages: + pkgdirurl = os.path.join(baseurl, package) + current = layout.checkout_url(pkgdirurl) + pristine = layout.checkout_url(pkgdirurl, pristine=True) if verbose: print "Checking package %s..." % package, sys.stdout.flush() - if not svn.exists(current): + if not svn.ls(current, noerror=1): if verbose: print "NO CURRENT" nocurrent.append(package) - elif not svn.exists(pristine): + elif not svn.ls(pristine, noerror=1): if verbose: print "NO PRISTINE" nopristine.append(package) @@ -356,19 +426,16 @@ def check_changed(pkgdirurl, all=0, show=0, verbose=0): "nocurrent": nocurrent, "nopristine": nopristine} -def checkout(pkgdirurl, path=None, revision=None, use_mirror=True): +def checkout(pkgdirurl, path=None, revision=None, branch=None, + distro=None): o_pkgdirurl = pkgdirurl - pkgdirurl = default_parent(o_pkgdirurl) - current = os.path.join(pkgdirurl, "current") + pkgdirurl = layout.package_url(o_pkgdirurl, distro=distro) + current = layout.checkout_url(pkgdirurl, branch=branch) if path is None: - _, path = os.path.split(pkgdirurl) - # if default_parent changed the URL, we can use mirrors because the - # user did not provided complete package URL - if (o_pkgdirurl != pkgdirurl) and use_mirror and mirror.enabled(): - current = mirror.checkout_url(current) - print "checking out from mirror", current + path = layout.package_name(pkgdirurl) + mirror.info(current) svn = SVN() - svn.checkout(current, path, revision=SVN.makerev(revision), show=1) + svn.checkout(current, path, rev=revision, show=1) def _getpkgtopdir(basedir=None): if basedir is None: @@ -378,15 +445,15 @@ def _getpkgtopdir(basedir=None): if dirname == "SPECS" or dirname == "SOURCES": topdir = os.pardir else: - topdir = "." + topdir = "" return topdir -def sync(dryrun=False): +def sync(dryrun=False, download=False): svn = SVN() topdir = _getpkgtopdir() # run svn info because svn st does not complain when topdir is not an # working copy - svn.info(topdir) + svn.info(topdir or ".") specsdir = os.path.join(topdir, "SPECS/") sourcesdir = os.path.join(topdir, "SOURCES/") for path in (specsdir, sourcesdir): @@ -401,18 +468,34 @@ def sync(dryrun=False): spec = rpm.TransactionSet().parseSpec(specpath) except rpm.error, e: raise Error, "could not load spec file: %s" % e - sources = [os.path.basename(name) - for name, no, flags in spec.sources()] - sourcesst = dict((os.path.basename(st.path), st) - for st in svn.status(sourcesdir, get_all=False, ignore=True)) + sources = dict((os.path.basename(name), name) + for name, no, flags in spec.sources()) + sourcesst = dict((os.path.basename(path), (path, st)) + for st, path in svn.status(sourcesdir, noignore=True)) toadd = [] - for source in sources: + for source, url in sources.iteritems(): sourcepath = os.path.join(sourcesdir, source) - if sourcesst.get(source): + pst = sourcesst.get(source) + if pst: + if os.path.isfile(sourcepath): + toadd.append(sourcepath) + else: + sys.stderr.write("warning: %s not found, skipping\n" % sourcepath) + elif download and not os.path.isfile(sourcepath): + print "%s not found, downloading from %s" % (sourcepath, url) + fmt = config.get("global", "download-command", + "wget -c -O '$dest' $url") + context = {"dest": sourcepath, "url": url} + try: + cmd = string.Template(fmt).substitute(context) + except KeyError, e: + raise Error, "invalid variable %r in download-command "\ + "configuration option" % e + execcmd(cmd, show=True) if os.path.isfile(sourcepath): toadd.append(sourcepath) else: - sys.stderr.write("warning: %s not found\n" % sourcepath) + raise Error, "file not found: %s" % sourcepath # rm entries not found in sources and still in svn found = os.listdir(sourcesdir) toremove = [] @@ -426,31 +509,35 @@ def sync(dryrun=False): for path in toremove: print "D\t%s" % path if not dryrun: - svn.remove(path) + svn.remove(path, local=True) for path in toadd: print "A\t%s" % path if not dryrun: - svn.add(path) + svn.add(path, local=True) -def commit(target=".", message=None): +def commit(target=".", message=None, logfile=None): svn = SVN() - status = svn.status(target, silent=True) + status = svn.status(target, quiet=True) if not status: print "nothing to commit" return - info = svn.info(target) - url = info.url + info = svn.info2(target) + url = info.get("URL") if url is None: raise Error, "working copy URL not provided by svn info" - mirrored = mirror.enabled(url) + mirrored = mirror.using_on(url) if mirrored: newurl = mirror.switchto_parent(svn, url, target) print "relocated to", newurl - # we can't use the svn object here because pexpect hides VISUAL - mopt = "" + # we can't use the svn object here because svn --non-interactive option + # hides VISUAL + opts = [] if message is not None: - mopt = "-m \"%s\"" % message - os.system("svn ci %s %s" % (mopt, target)) + opts.append("-m \"%s\"" % message) + if logfile is not None: + opts.append("-F \"%s\"" % logfile) + mopts = " ".join(opts) + os.system("svn ci %s %s" % (mopts, target)) if mirrored: print "use \"repsys switch\" in order to switch back to mirror "\ "later" @@ -458,8 +545,10 @@ def commit(target=".", message=None): def switch(mirrorurl=None): svn = SVN() topdir = _getpkgtopdir() - info = svn.info(topdir) - wcurl = info.url + info = svn.info2(topdir) + wcurl = info.get("URL") + if wcurl is None: + raise Error, "working copy URL not provided by svn info" newurl = mirror.autoswitch(svn, topdir, wcurl, mirrorurl) print "switched to", newurl @@ -483,31 +572,43 @@ def get_submit_info(path): if not os.path.isdir(os.path.join(path, ".svn")): raise Error, "subversion directory not found" - # Now, extract the package name. svn = SVN() - info = svn.info(path) - url = info.url + # Now, extract the package name. + info = svn.info2(path) + url = info.get("URL") + if url is None: + raise Error, "missing URL from svn info %s" % path toks = url.split("/") if len(toks) < 2 or toks[-1] != "current": - raise Error, "invalid package URL %s, needs current/" % url + raise Error, "unexpected URL received from 'svn info'" name = toks[-2] + url = "/".join(toks[:-1]) # Finally, guess revision. max = -1 files = [] files.extend(glob.glob("%s/*" % specsdir)) files.extend(glob.glob("%s/*" % sourcesdir)) - for file in files: - info = svn.info(file) - if info is None: # not in working copy + for file in files: + try: + info = svn.info2(file) + except Error: + # possibly not tracked continue - rev = info.commit_revision.number - if rev > max: - max = rev + if info is None: + continue + rawrev = info.get("Last Changed Rev") + if rawrev: + rev = int(rawrev) + if rev > max: + max = rev if max == -1: - raise Error, "unable to find the latest revision" + raise Error, "revision tag not found in 'svn info' output" + + if mirror.using_on(url): + url = mirror.switchto_parent_url(url) - return name, max + return name, url, max # vim:et:ts=4:sw=4 diff --git a/RepSys/svn.py b/RepSys/svn.py index 527b1e5..985329d 100644 --- a/RepSys/svn.py +++ b/RepSys/svn.py @@ -1,18 +1,10 @@ from RepSys import Error, config from RepSys.util import execcmd, get_auth - import sys -import os import re import time -import threading -import tempfile -import pysvn - -__all__ = ["SVN", "Revision", "SVNLogEntry", "SVNError"] -class SVNError(Error): - pass +__all__ = ["SVN", "SVNLook", "SVNLogEntry"] class SVNLogEntry: def __init__(self, revision, author, date): @@ -26,208 +18,347 @@ class SVNLogEntry: return cmp(self.date, other.date) class SVN: - _client = None - _client_lock = None - _current_message = None - - def __init__(self): - self._client = pysvn.Client() - self._client_lock = threading.Lock() - self._client.callback_get_log_message = self._log_handler - self._client.callback_get_login = self._unsupported_auth - self._client.callback_ssl_client_cert_password_prompt = \ - self._unsupported_auth - self._client.callback_ssl_client_cert_prompt = \ - self._unsupported_auth - self._client.callback_ssl_server_prompt = \ - self._unsupported_auth - - def _log_handler(self): - if self._current_message is None: - #TODO make it use EDITOR - raise ValueError, "No log message defined" - return True, self._current_message - - def _unsupported_auth(self, *args, **kwargs): - raise SVNError, "svn is trying to get login information, " \ - "seems that you're not using ssh-agent" - - def _get_log_message(self, received_kwargs): - message = received_kwargs.pop("log", None) - messagefile = received_kwargs.pop("logfile", None) - if messagefile and not message: - message = open(messagefile).read() - return message - - def _set_notify_callback(self, callback): - self._client.callback_notify = callback - - def _make_wrapper(self, meth, notify=None): - def wrapper(*args, **kwargs): - self._client_lock.acquire() - try: - self._current_message = self._get_log_message(kwargs) - ignore_errors = kwargs.pop("noerror", None) - if notify: - self._client.callback_notify = notify - try: - return meth(*args, **kwargs) - except pysvn.ClientError, (msg,): - if not ignore_errors: - raise SVNError, msg - return None - finally: - self._client_lock.release() - self._current_message = None - return wrapper - - def _client_wrap(self, attrname): - meth = getattr(self._client, attrname) - wrapper = self._make_wrapper(meth) - return wrapper - - def __getattr__(self, attrname): - return self._client_wrap(attrname) - - def makerev(number=None, head=None): - if number is not None: - args = (pysvn.opt_revision_kind.number, number) - else: - args = (pysvn.opt_revision_kind.head,) - return pysvn.Revision(*args) - makerev = staticmethod(makerev) - - def revision(self, url, last_changed=False): - infos = self._client.info2(url, recurse=False) - if last_changed: - revnum = infos[0][1].last_changed_rev.number - else: - revnum = infos[0][1].rev.number - return revnum - - # this override method fixed the problem in pysvn's mkdir which - # requires a log_message parameter - def mkdir(self, path, log=None, **kwargs): - meth = self._client_wrap("mkdir") - # we can't raise an error because pysvn's mkdir will use - # log_message only if path is remote, but it *always* requires this - # parameter. Also, 'log' is never used. - log = log or "There's a silent bug in your code" - return meth(path, log, log=None, **kwargs) - - def checkout(self, url, targetpath, show=False, **kwargs): - if show: - def callback(event): - types = pysvn.wc_notify_action - action = event["action"] - if action == types.update_add: - print "A %s" % event["path"] - elif action == types.update_completed: - print "Checked out revision %d" % \ - event["revision"].number - self._set_notify_callback(callback) - meth = self._client_wrap("checkout") - meth(url, targetpath, **kwargs) - - def checkin(self, path, log, **kwargs): - # XXX use EDITOR when log empty - meth = self._client_wrap("checkin") - return meth(path, log, log=None, **kwargs) - - def log(self, *args, **kwargs): - meth = self._client_wrap("log") - entries = meth(discover_changed_paths=True, *args, **kwargs) - if entries is None: - return - for entrydic in entries: - entry = SVNLogEntry(entrydic["revision"].number, - entrydic["author"], - time.localtime(entrydic["date"])) - entry.lines[:] = entrydic["message"].split("\n") - for cp in entrydic["changed_paths"]: - from_rev = cp["copyfrom_revision"] - if from_rev: - from_rev = from_rev.number - changed = { - "action": cp["action"], - "path": cp["path"], - "from_rev": from_rev, - "from_path": cp["copyfrom_path"], - } - entry.changed.append(changed) - yield entry - - def exists(self, path): - return self.ls(path, noerror=1) is not None - - def status(self, *args, **kwargs): - # add one keywork "silent" that workaround the strange behavior of - # pysvn's get_all, which seems to be broken, this way we also have - # the same interface of svn.py from repsys 1.6.x - meth = self._client_wrap("status") - silent = kwargs.pop("silent", None) - st = meth(*args, **kwargs) - if silent: - unversioned = pysvn.wc_status_kind.unversioned - st = [entry for entry in st - if entry.text_status is not unversioned] - return st - - def diff(self, path1, *args, **kwargs): - head = pysvn.Revision(pysvn.opt_revision_kind.head) - revision1 = kwargs.pop("revision1", head) - revision2 = kwargs.pop("revision2", head) - if args: - kwargs["url_or_path2"] = args[0] - tmpdir = tempfile.gettempdir() - meth = self._client_wrap("diff") - diff_text = meth(tmpdir, path1, revision1=revision1, - revision2=revision2, **kwargs) - return diff_text - - def _edit_message(self, message): - # argh! - editor = os.getenv("EDITOR", "vim") - fd, fpath = tempfile.mkstemp(prefix="repsys") - result = (False, None) + def _execsvn(self, *args, **kwargs): + localcmds = ("add", "revert", "cleanup") + if not kwargs.get("show") and args[0] not in localcmds: + args = list(args) + args.append("--non-interactive") + svn_command = config.get("global", "svn-command", + "SVN_SSH='ssh -o \"BatchMode yes\"' svn") + cmdstr = svn_command + " " + " ".join(args) try: - f = os.fdopen(fd, "w") - f.write(message) - f.close() - lastchange = os.stat(fpath).st_mtime - for i in xrange(10): - status = os.system("%s %s" % (editor, fpath)) - if status != 0: - raise SVNError, "the editor failed with %d" % status - newchange = os.stat(fpath).st_mtime - if newchange == lastchange: - print "Log message unchanged or not specified" - print "(a)bort, (c)ontinue, (e)dit" - choice = raw_input() - if not choice or choice[0] == 'e': - continue - elif choice[0] == 'a': - break - elif choice[0] == 'c': - pass # ignore and go ahead - newmessage = open(fpath).read() - result = (True, newmessage) - break - finally: - os.unlink(fpath) - return result - - def propedit(self, propname, pkgdirurl, revision, revprop=False): - revision = (revision) - if revprop: - propget = self.revpropget - propset = self.revpropset + return execcmd(cmdstr, **kwargs) + except Error, e: + if "Permission denied" in e.message: + raise Error, ("%s\n" + "Seems ssh-agent or ForwardAgent are not setup, see " + "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" + " for more information." % e) + elif "authorization failed" in e.message: + raise Error, ("%s\n" + "Note that repsys does not support any HTTP " + "authenticated access." % e) + raise + + def _execsvn_success(self, *args, **kwargs): + status, output = self._execsvn(*args, **kwargs) + return status == 0 + + def _add_log(self, cmd_args, received_kwargs, optional=0): + if (not optional or + received_kwargs.has_key("log") or + received_kwargs.has_key("logfile")): + ret = received_kwargs.get("log") + if ret is not None: + cmd_args.append("-m '%s'" % ret) + ret = received_kwargs.get("logfile") + if ret is not None: + cmd_args.append("-F '%s'" % ret) + + def _add_revision(self, cmd_args, received_kwargs, optional=0): + if not optional or received_kwargs.has_key("rev"): + ret = received_kwargs.get("rev") + if isinstance(ret, basestring): + try: + ret = int(ret) + except ValueError: + raise Error, "invalid revision provided" + if ret: + cmd_args.append("-r %d" % ret) + + def add(self, path, **kwargs): + cmd = ["add", path] + return self._execsvn_success(noauth=1, *cmd, **kwargs) + + def copy(self, pathfrom, pathto, **kwargs): + cmd = ["copy", pathfrom, pathto] + self._add_revision(cmd, kwargs, optional=1) + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def remove(self, path, force=0, **kwargs): + cmd = ["remove", path] + self._add_log(cmd, kwargs) + if force: + cmd.append("--force") + return self._execsvn_success(*cmd, **kwargs) + + def mkdir(self, path, **kwargs): + cmd = ["mkdir", path] + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def commit(self, path, **kwargs): + cmd = ["commit", path] + self._add_log(cmd, kwargs) + return self._execsvn_success(*cmd, **kwargs) + + def export(self, url, targetpath, **kwargs): + cmd = ["export", "'%s'" % url, targetpath] + self._add_revision(cmd, kwargs, optional=1) + return self._execsvn_success(*cmd, **kwargs) + + def checkout(self, url, targetpath, **kwargs): + cmd = ["checkout", "'%s'" % url, targetpath] + self._add_revision(cmd, kwargs, optional=1) + return self._execsvn_success(*cmd, **kwargs) + + def propset(self, propname, value, targets, **kwargs): + cmd = ["propset", propname, "'%s'" % value, targets] + return self._execsvn_success(*cmd, **kwargs) + + def propedit(self, propname, target, **kwargs): + cmd = ["propedit", propname, target] + if kwargs.get("rev"): + cmd.append("--revprop") + self._add_revision(cmd, kwargs) + return self._execsvn_success(local=True, show=True, *cmd, **kwargs) + + def revision(self, path, **kwargs): + cmd = ["info", path] + status, output = self._execsvn(local=True, *cmd, **kwargs) + if status == 0: + for line in output.splitlines(): + if line.startswith("Last Changed Rev: "): + return int(line.split()[3]) + return None + + def info(self, path, **kwargs): + cmd = ["info", path] + status, output = self._execsvn(local=True, *cmd, **kwargs) + if status == 0 and "Not a versioned resource" not in output: + return output.splitlines() + return None + + def info2(self, *args, **kwargs): + lines = self.info(*args, **kwargs) + if lines is None: + return None + pairs = [[w.strip() for w in line.split(":", 1)] for line in lines] + info = dict(pairs) + return info + + def ls(self, path, **kwargs): + cmd = ["ls", path] + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output.split() + return None + + def status(self, path, **kwargs): + cmd = ["status", path] + if kwargs.get("verbose"): + cmd.append("-v") + if kwargs.get("noignore"): + cmd.append("--no-ignore") + if kwargs.get("quiet"): + cmd.append("--quiet") + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.splitlines()] + return None + + def cleanup(self, path, **kwargs): + cmd = ["cleanup", path] + return self._execsvn_success(*cmd, **kwargs) + + def revert(self, path, **kwargs): + cmd = ["revert", path] + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def switch(self, url, oldurl=None, path=None, relocate=False, **kwargs): + cmd = ["switch"] + if relocate: + if oldurl is None: + raise Error, "You must supply the old URL when "\ + "relocating working copies" + cmd.append("--relocate") + cmd.append(oldurl) + cmd.append(url) + if path is not None: + cmd.append(path) + return self._execsvn_success(*cmd, **kwargs) + + def update(self, path, **kwargs): + cmd = ["update", path] + self._add_revision(cmd, kwargs, optional=1) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def merge(self, url1, url2=None, rev1=None, rev2=None, path=None, + **kwargs): + cmd = ["merge"] + if rev1 and rev2 and not url2: + cmd.append("-r") + cmd.append("%s:%s" % (rev1, rev2)) + cmd.append(url1) else: - propget = self.propget - propset = self.propset - revision, message = propget(propname, pkgdirurl, revision=revision) - changed, newmessage = self._edit_message(message) - if changed: - propset(propname, newmessage, pkgdirurl, revision=revision) + if not url2: + raise ValueError, \ + "url2 needed if two revisions are not provided" + if rev1: + cmd.append("%s@%s" % (url1, rev1)) + else: + cmd.append(url1) + if rev2: + cmd.append("%s@%s" % (url2, rev2)) + else: + cmd.append(url2) + if path: + cmd.append(path) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return [x.split() for x in output.split()] + return None + + def diff(self, pathurl1, pathurl2=None, **kwargs): + cmd = ["diff", pathurl1] + self._add_revision(cmd, kwargs, optional=1) + if pathurl2: + cmd.append(pathurl2) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output + return None + + def cat(self, url, **kwargs): + cmd = ["cat", url] + self._add_revision(cmd, kwargs, optional=1) + status, output = self._execsvn(*cmd, **kwargs) + if status == 0: + return output + return None + + def log(self, url, start=None, end=0, limit=None, **kwargs): + cmd = ["log", "-v", url] + if start is not None or end != 0: + if start is not None and type(start) is not type(0): + try: + start = int(start) + except (ValueError, TypeError): + raise Error, "invalid log start revision provided" + if type(end) is not type(0): + try: + end = int(end) + except (ValueError, TypeError): + raise Error, "invalid log end revision provided" + start = start or "HEAD" + cmd.append("-r %s:%s" % (start, end)) + if limit is not None: + try: + limit = int(limit) + except (ValueError, TypeError): + raise Error, "invalid limit number provided" + cmd.append("--limit %d" % limit) + status, output = self._execsvn(*cmd, **kwargs) + if status != 0: + return None + + revheader = re.compile("^r(?P<revision>[0-9]+) \| (?P<author>[^\|]+) \| (?P<date>[^\|]+) \| (?P<lines>[0-9]+) (?:line|lines)$") + changedpat = re.compile(r"^\s+(?P<action>[^\s]+) (?P<path>[^\s]+)(?: \([^\s]+ (?P<from_path>[^:]+)(?:\:(?P<from_rev>[0-9]+))?\))?$") + logseparator = "-"*72 + linesleft = 0 + entry = None + log = [] + appendchanged = 0 + changedheader = 0 + for line in output.splitlines(): + line = line.rstrip() + if changedheader: + appendchanged = 1 + changedheader = 0 + elif appendchanged: + if not line: + appendchanged = 0 + continue + m = changedpat.match(line) + if m: + changed = m.groupdict().copy() + from_rev = changed.get("from_rev") + if from_rev is not None: + try: + changed["from_rev"] = int(from_rev) + except (ValueError, TypeError): + raise Error, "invalid revision number in svn log" + entry.changed.append(changed) + elif linesleft == 0: + if line != logseparator: + m = revheader.match(line) + if m: + linesleft = int(m.group("lines")) + timestr = " ".join(m.group("date").split()[:2]) + timetuple = time.strptime(timestr, + "%Y-%m-%d %H:%M:%S") + entry = SVNLogEntry(int(m.group("revision")), + m.group("author"), timetuple) + log.append(entry) + changedheader = 1 + else: + entry.lines.append(line) + linesleft -= 1 + log.sort() + log.reverse() + return log + +class SVNLook: + def __init__(self, repospath, txn=None, rev=None): + self.repospath = repospath + self.txn = txn + self.rev = rev + + def _execsvnlook(self, cmd, *args, **kwargs): + execcmd_args = ["svnlook", cmd, self.repospath] + self._add_txnrev(execcmd_args, kwargs) + execcmd_args += args + execcmd_kwargs = {} + keywords = ["show", "noerror"] + for key in keywords: + if kwargs.has_key(key): + execcmd_kwargs[key] = kwargs[key] + return execcmd(*execcmd_args, **execcmd_kwargs) + + def _add_txnrev(self, cmd_args, received_kwargs): + if received_kwargs.has_key("txn"): + txn = received_kwargs.get("txn") + if txn is not None: + cmd_args += ["-t", txn] + elif self.txn is not None: + cmd_args += ["-t", self.txn] + if received_kwargs.has_key("rev"): + rev = received_kwargs.get("rev") + if rev is not None: + cmd_args += ["-r", rev] + elif self.rev is not None: + cmd_args += ["-r", self.rev] + + def changed(self, **kwargs): + status, output = self._execsvnlook("changed", **kwargs) + if status != 0: + return None + changes = [] + for line in output.splitlines(): + line = line.rstrip() + if not line: + continue + entry = [None, None, None] + changedata, changeprop, path = None, None, None + if line[0] != "_": + changedata = line[0] + if line[1] != " ": + changeprop = line[1] + path = line[4:] + changes.append((changedata, changeprop, path)) + return changes + + def author(self, **kwargs): + status, output = self._execsvnlook("author", **kwargs) + if status != 0: + return None + return output.strip() # vim:et:ts=4:sw=4 diff --git a/RepSys/util.py b/RepSys/util.py index 8c66199..83f2ebe 100644 --- a/RepSys/util.py +++ b/RepSys/util.py @@ -29,7 +29,8 @@ def execcmd(*cmd, **kwargs): status = os.system(cmdstr) output = "" else: - status, output = commands_getstatusoutput("LANG=C LANGUAGE=C LC_ALL=C "+cmdstr) + status, output = commands_getstatusoutput( + "LANG=C LANGUAGE=C LC_ALL=C "+cmdstr) if status != 0 and not kwargs.get("noerror"): raise Error, "command failed: %s\n%s\n" % (cmdstr, output) if config.getbool("global", "verbose", 0): |