From d62a7f9774d9df9ce29e56dbecbab2f520fc3be5 Mon Sep 17 00:00:00 2001 From: Nicolas Vigier Date: Thu, 6 Jan 2011 14:51:53 +0000 Subject: To avoid confusion, Mageia repsys fork is being renamed to mgarepo --- BRANCH | 419 +++++++++++++++++++++++ CHANGES | 163 +++++++++ MANIFEST.in | 10 + README | 12 + README.LDAP | 61 ++++ RepSys/ConfigParser.py | 434 +++++++++++++++++++++++ RepSys/__init__.py | 19 ++ RepSys/binrepo.py | 393 +++++++++++++++++++++ RepSys/cgi/__init__.py | 0 RepSys/cgi/soapserver.py | 93 +++++ RepSys/cgi/submit.py | 119 +++++++ RepSys/cgi/xmlrpcserver.py | 111 ++++++ RepSys/cgiutil.py | 53 +++ RepSys/command.py | 61 ++++ RepSys/commands/__init__.py | 0 RepSys/commands/authoremail.py | 37 ++ RepSys/commands/changed.py | 41 +++ RepSys/commands/ci.py | 35 ++ RepSys/commands/co.py | 67 ++++ RepSys/commands/create.py | 34 ++ RepSys/commands/del.py | 30 ++ RepSys/commands/editlog.py | 39 +++ RepSys/commands/getspec.py | 38 +++ RepSys/commands/getsrpm.py | 100 ++++++ RepSys/commands/log.py | 62 ++++ RepSys/commands/markrelease.py | 103 ++++++ RepSys/commands/patchspec.py | 38 +++ RepSys/commands/putsrpm.py | 59 ++++ RepSys/commands/rpmlog.py | 68 ++++ RepSys/commands/submit.py | 211 ++++++++++++ RepSys/commands/switch.py | 33 ++ RepSys/commands/sync.py | 38 +++ RepSys/commands/up.py | 22 ++ RepSys/commands/upload.py | 28 ++ RepSys/layout.py | 207 +++++++++++ RepSys/log.py | 633 ++++++++++++++++++++++++++++++++++ RepSys/mirror.py | 129 +++++++ RepSys/plugins/__init__.py | 27 ++ RepSys/plugins/ldapusers.py | 189 ++++++++++ RepSys/plugins/sample.py.txt | 14 + RepSys/rpmutil.py | 759 +++++++++++++++++++++++++++++++++++++++++ RepSys/simplerpm.py | 19 ++ RepSys/svn.py | 430 +++++++++++++++++++++++ RepSys/util.py | 141 ++++++++ create-srpm | 126 +++++++ default.chlog | 41 +++ repsys | 93 +++++ repsys-example.conf | 74 ++++ repsys-ssh | 2 + repsys.8 | 218 ++++++++++++ repsys.conf | 18 + revno.chlog | 41 +++ setup.cfg | 3 + setup.py | 33 ++ 54 files changed, 6228 insertions(+) create mode 100644 BRANCH create mode 100644 CHANGES create mode 100644 MANIFEST.in create mode 100644 README create mode 100644 README.LDAP create mode 100644 RepSys/ConfigParser.py create mode 100644 RepSys/__init__.py create mode 100644 RepSys/binrepo.py create mode 100644 RepSys/cgi/__init__.py create mode 100644 RepSys/cgi/soapserver.py create mode 100644 RepSys/cgi/submit.py create mode 100644 RepSys/cgi/xmlrpcserver.py create mode 100644 RepSys/cgiutil.py create mode 100644 RepSys/command.py create mode 100644 RepSys/commands/__init__.py create mode 100644 RepSys/commands/authoremail.py create mode 100644 RepSys/commands/changed.py create mode 100644 RepSys/commands/ci.py create mode 100644 RepSys/commands/co.py create mode 100644 RepSys/commands/create.py create mode 100644 RepSys/commands/del.py create mode 100644 RepSys/commands/editlog.py create mode 100644 RepSys/commands/getspec.py create mode 100644 RepSys/commands/getsrpm.py create mode 100644 RepSys/commands/log.py create mode 100644 RepSys/commands/markrelease.py create mode 100644 RepSys/commands/patchspec.py create mode 100644 RepSys/commands/putsrpm.py create mode 100644 RepSys/commands/rpmlog.py create mode 100644 RepSys/commands/submit.py create mode 100644 RepSys/commands/switch.py create mode 100644 RepSys/commands/sync.py create mode 100644 RepSys/commands/up.py create mode 100644 RepSys/commands/upload.py create mode 100644 RepSys/layout.py create mode 100644 RepSys/log.py create mode 100644 RepSys/mirror.py create mode 100644 RepSys/plugins/__init__.py create mode 100644 RepSys/plugins/ldapusers.py create mode 100644 RepSys/plugins/sample.py.txt create mode 100644 RepSys/rpmutil.py create mode 100644 RepSys/simplerpm.py create mode 100644 RepSys/svn.py create mode 100644 RepSys/util.py create mode 100755 create-srpm create mode 100644 default.chlog create mode 100755 repsys create mode 100644 repsys-example.conf create mode 100755 repsys-ssh create mode 100644 repsys.8 create mode 100644 repsys.conf create mode 100644 revno.chlog create mode 100644 setup.cfg create mode 100755 setup.py diff --git a/BRANCH b/BRANCH new file mode 100644 index 0000000..ebb87c7 --- /dev/null +++ b/BRANCH @@ -0,0 +1,419 @@ +================================ +The detached binaries repository +================================ + +.. contents:: + +A brief description +=================== + +Ideally, all binaries from packages sources (ie. all the binary files inside +SOURCES/) will be placed in another subversion repository. This repository +is called "tarballs repository", "binaries repository" or just "binrepo". +It will contain mostly the same directory structure of the main repository, +but instead of having SOURCES and SPECS, it will only have a SOURCES +directory. Every copy/move operation should happen in both repositories. + +In order to allow deceasing binaries from older distributions, each stable +distro will have its own subversion repository for binary files. repsys +knows how to access these binrepos by checking which URL defined in the +"[binrepo]" section of the configuration file matches the path-part of the +repository being accessed. (see open issues) + +The package changelogs will be generated from SVN commit logs in the main +"plaintext" repository ("txtrepo" for short) only. Old changelogs will be +preserved, as even empty revisions are preserved in the binaries-filtering +conversion. + + +Mapping repositories states +--------------------------- + +In order to allow the use of `repsys {getsrpm,co} -r REV`, repsys will have +to use a reference in the text repo which will be used to know in what +state was the binrepo when a binary was uploaded. + +We cannot use direct revision number mapping through properties/files/etc +mainly because we may have multiple binaries repositories, and eventually +they can be filtered for reducing space, thus can't ensure revisions will +survive. Thus another mechanism which relies on dates instead of revisions +numbers is needed. + +When a binary is uploaded to the binrepo, the file `sha1.lst` is updated to +have the files's hash and commited in the main text repo. This file will be +used as the reference when the user uses -r REV on repsys. repsys will +checkout the package in the main text repo with -r REV and then will use +the "Last Changed Date" of `sha1.lst` to checkout the binrepo part. Thus, +`sha1.lst` should be always commited to the main text repository *after* the +corresponding binary files have been commited to the binrepo. Hooks in the +main repository may be used to try to enforce this, by checking if the files +changed in `sha1.lst` are already commited in the corresponding binrepo. + +Computation of `sha1.lst` is unlikely to be an issue: + +- it should not happen too often for any given package +- it takes[0] less than 10s to sha1sum all SOURCES of openoffice.org-3.1-1mdv2010.0.src.rpm +- it probably takes way less than the time to upload the file into the repository +- it can be computed in parallel to the binrepo commit, and probably finish + before that, thus ready by the time `sha1.lst` should be commited +- users don't need to verify the SHA1s "everytime", but the build system + does, thus Repsys can default to not verify and avoid wasting users' time + +The use of `sha1.lst` has the valuable property of tying the state of the main +repository and the binrepo. With it, at getsrpm time of a package +submission we can verify the SHA1 of the SOURCES-bin, and be sure that +either the package will be built with the expected state, or early fail the +build. It also allows for verifying binaries without trusting the binrepo, +which may be useful if we consider using an unversioned plain filesystem +storage in the future (for old distros or whatever), or at "client side", +which maintainers may find useful. + +[0]: In a single core AMD Athlon(tm) 3800+ (2400Mhz) + +Mapping of revisions using SVN properties +----------------------------------------- + +Alternatively to using the above "sha1.lst scheme", the revision mapping +between the main repository and a binrepo could be done using subversion +properties. This could be done by making every commit to binrepos also +cause a corresponding commit in the main text repository to happen, which +would update a property recording the current date. That is, a subversion +property in the main text repository would be kept, such that for any given +main repository revision, the corresponding state of the binrepos is +obtainable (using the registered date). + +This would be "more transparent", as it can be maintened simply by using +subversion hooks, without user intervention. OTOH, as every time the user +commits to a binrepo this would result in a commit in the main repository, +it would require the user to "svn up" the directories from there before +commiting, after every binrepo commit. Also, this might result in a big +number of "bogus" commits to the main repository, which could be seen as log +pollution, and may potentially increase space usage etc.. + +Why a new repository without the tarballs +========================================== + +- the current svn repository is too large, hard to manage +- big binary files (in general, "tarballs") history is of little value in + the distro development, we care much more about our specs, patches, + configurations, etc.; nonetheless, those big files we don't care much for + take the most resources and make backups and restoration in case of + failure very expensive, much more so than the more valuable data +- there is no easy way to strip undesired tarballs without recreating the + whole repository +- fedora and ubuntu have separated repositories, so we must have it too! + +Numbers +------- + +Current repository is +390000 revisions and ~340Gb big, while the bzip2ed +dumps backup for it takes about a bit more than half that size (FIXME: +estimative, can't check in the backup server right now). Current txtrepo +with the same number of revisions is ~180Gb big, takes about 2-3 days to be +imported, while the gzipped full dump backup for it currently takes ~1.2Gb. +Initial binrepo for Cooker (only `current/` packages' branches) took ~28Gb +in disk, gzipped full dump for it takes ~25Gb, took about 5h30m to be +populated from the current in use repository ("oldrepo"). + + +Drawbacks of this layout +========================= + +- (always) everything that changes the single-repository usage increases the chance + of failure and make things more complicated. +- subversion can't be used alone as easily as the current scheme allows +- copying binaries between distro branches may not be "svn-cheap" anymore + (unless they're in the same binrepo) +- ... + + +Open issues +============ + +Multiple binrepos dont allow us to have one permanent URL +--------------------------------------------------------- + +We would have to update the configuration files from all the users in order +to add a new stable repository. spuk suggests to use properties in the main +text repo that would point to the right repository locations. + +How to handle failures when operating on more repositores? +---------------------------------------------------------- + +binrepos should replicate the structure of the main text repo. What we +should do if the markrelease succeeds in the binrepo, but fails in the main +text repo? + +R: Markrelease must be done first in the txtrepo. If it fails there "we're +in trouble" (though currently, we just miss it[0]). When the markrelease is +done in the txtrepo, we can do markrelease in the binrepo using '-r {DATE}', +using the markrelease date in the txtrepo as '{DATE}'. + +[0] We should add transaction support for markrelease. The transaction could +be stored out of the packages SVN (another SVN, a DB, a txt file, etc.), and +would work like: + +0. mark beginning of markrelease, early failing the package build if it fails +1. do markrelease +2. mark sucessful end of markrelease + or mark failed markrelease, so we can replay it later + + +Interesting use cases (first phase) +=================================== + +repsys co 2008.1/mutt +--------------------- + +- repsys checkouts + http://svn.mandriva.com/svn/packages/updates/2008.1/mutt/current to the + mutt directory + +- repsys checkouts + http://svn.mandriva.com/svn/binrepo/updates/2008.1/mutt/current/SOURCES + into mutt/SOURCES-bin + +- creates symlinks for all files found in SOURCES-bin/ into ../SOURCES/ + + (rpm doesn't handle symlinks, this allows us to have explicit links and + proper src.rpm generates by rpmbuild) + +In case the path doesn't exist in the binrepo it will not fail, as we may +have not imported all packages or the repository is not prepared to work on +this model, etc. + +markrelease of a package +------------------------ + +:: + + $ repsys markrelease + +- will copy current/ to releases/VERSION/RELEASE, as usual + +- will copy current/ to releases/, on the binrepo too + +Optionally, markrelease could create revprops indicating which is the +revision of current/ on the binrepo that represents the tarballs that are +being tagged. + + +Use cases to be implemented after the first phase +================================================= + +upgrading to a newer version of the package +------------------------------------------- + +:: + + $ cd bla/SOURCES/ + $ wget http://prdownloads.sourceforge.net/bla/bla-1.6.tar.bz2 + $ repsys add bla-1.6.0.tar.bz2 + +- repsys notices this is a tarball (checking filename and/or file size) + +- repsys will move the file to SOURCES-bin/, create the symlink, and svn-add + it to the working copy + + $ # the user updates the spec + + $ repsys rm SOURCES/bla-1.5.1.tar.bz2 + +- it will remove the symlink and run svn rm on + SOURCES-bin/bla-1.6.0.tar.bz2:: + + $ cd ../ # package top dir + $ repsys ci + +- repsys will commit the new tarball on SOURCES-bin/ and then on the rest + of the working copy + +repsys sync would perform these steps too. + +importing a package +------------------- + + $ repsys putsrpm mypkg.src.rpm + +- repsys will open the src.rpm + +- will look for tarballs inside SOURCES/ and import them to + http://svn.mandriva.com/svn/binrepo/cooker/mypkg/current/SOURCES/ + +- will move the tarballs out of SOURCES and import the remaining files to + http://svn.mandriva.com/svn/packages/cooker/mypkg/current/ + +- will do whatever else putsrpm already does + +TODO +===== + +First phase +----------- + +- upload +- markrelease +- putsrpm +- getsrpm + + +Second phase +------------ + +- up +- sync + +Rejected or postponed ideas +=========================== + +Use of a plain filesystem storage for the tarballs +-------------------------------------------------- + +This was planned, then rejected. It becomes too complicated when thinking +about markrelease, and mapping SVN revisions in the main repository to +binaries versions in the "tarballs storage", basically requiring +implementing VCS-like features on top of filesystem. Would also require +implementing another authentication and access scheme. The main feature +would be ease of removing old binaries, which isn't much of a point because +we don't know precisely what and when we want to remove, so may end up not +removing much files anyway. + +Use of a plain unversioned filesystem storage for the tarballs +-------------------------------------------------------------- + +Different than the previous one, this would mean not relying at all on +binary files history keeping. Structure could be something simple like:: + + packages/${pkg:0:1}/$pkg/$tarball + +This alternative does not suffice for Cooker, nor for supported distros, for +which we want history. It could, however, at some point be used for "very +old" distros, for which we may have lost interest in keeping *binaries* +history (package history will kept "forever" in the main SVN repository). +Alternatively, "resetting" an SVN binrepo (i.e. recreate the repository) to +contain only the latest tarballs would probably take about the same amount +of space, anyway... + +Open tarballs repository +------------------------ + +This idea is not really rejected. It does not go against splitting txtrepo +and binrepo, but rather complement this idea, where the +open-tarballs-repository would take the place of the binrepo. The txtrepo +would still be used +- the same way. This repository could be used +selectively, for packages where it makes sense, while most packages could be +kept "closed", still as tarballs. + +Use of externals for more seamless Subversion usage +--------------------------------------------------- + +This idea is not discarded, but it just provides easiness. OTOH, it makes +things more complicated: + +- markrelease: externals would have to be updated in order to make it point + to the tagged version in the binrepo, otherwise changes in + current@binrepo would change older releases; +- branching whole distro: even though subversion now supports "relative + externals", we would have to update the URLs for *every* package on the + distro, as the path to reach the binrepo spans the local distribution + directory; +- keeping externals up-to-date (as stated above and below) +- authentication and access control: only markrelease action done by the + build system should be allowed to change externals (so what about importing + new packages?) +- just a convenience, we don't need and shouldn't rely on externals for + running the build system, while most people will use the repositories via + Repsys, so why spend time to implement and keep it? +- "svn co" works transparently, cool, but "svn co -r N" does not, otherwise + every change in the binrepo would require svn:externals to be updated in + the respective package; +- it does not solve the problem of creating and handling symlinks between + SOURCES and SOURCES-bin. + +Keeping svn:externals updated for every package has almost the same cost of +keeping the `sha1.lst` updated, with the difference that in the latter we +would not have to update every package when creating distro branches. + +Use of "external" xdelta to save space on binaries +-------------------------------------------------- + +But how? First idea is this could be done by defining a protocol and +assuming repository manipulation with repsys (for ease). Repsys could +xdelta tarballs and add it to SVN with a special filename, then use it when +checking out. Would require a policy/algorithm on when to ditch old whole +binaries, too (i.e. hopefully wouldn't need to be handled manually by the +maintainer). Also, this is something complemental to splitting the +repository, so we may do it later, for binrepos. + + +The Future +========== + +- Open tarballs repositories + + - suited for GIT, maybe multi-VCS + - incremental move + - not everything will be suited for this, must handle all cases or be + optional + +- Xdelta + + +Deployment +========== + +The current repository will be kept around for a while, in readonly state. +Initial binrepos will be populated with the binaries in the `current/` +branches of packages. + +The binrepo mappings config might be kept in a fixed subversion revision +property (revision 0?). + +Rough steps +----------- + +- check for agreement between subversion repository filters for binaries, + and repsys +- upgrade repsys everywhere + + - kenobi + - cluster nodes + - raoh + - titan + +- populate the binrepos for each supported distro, from a specific revision + of oldrepo, and mass commmit the corresponding `sha1.lst` in txtrepo for + every package + + - set svn:date revprop of the `sha1.lst` mass commit to the date of the + oldrepo revision + - before mass commiting the `sha1.lst`, possibly freeze oldrepo, check + for changes to sources after the selected revision, and update the + binrepo as necessary + +- check Secteam scripts, make needed changes to get them ready (non + critical) +- set up the new repositories + + - hook for filtering of disallowed (binary) files in main repository + - binrepos mappings + +- make the new main + binrepos repositories available, but readonly + + - keep new main repository in sync with the old repository with hooks + +- make current repository readonly and enable verification of sha1.lst at + package submission time + +- make sure new main repository and old repository are in sync + + - resync binrepos with the old repository as needed + +- final tests + + - change something + - submit + - etc. + +- make the new repositories writeable + diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..587b33d --- /dev/null +++ b/CHANGES @@ -0,0 +1,163 @@ +* Development +- added support to "binary repository" +- added new commands upload, up and del to help handling tarballs in the + binaries repository + +* 1.9 +- really fixed -M +- new command log: shows the svn log for a given package +- added option -s to co, to allow checking out only SPECS +- use a better message when checking out from the mirror +- show the path where getspec wrote the spec file + +* 1.8 +- make the -M option work again +- sync now adds the spec file (pointed by blino) +- fixed putsrpm to not create old log files for packages without changelog +- submit now can fetch the revision number if not specified (it also shows + the author and the first line of the commit) +- allow submitting many packages at once, even without support on server + side (the same behavior as running repsys submit for each package) +- append a parameter sid=UUID for every set of packages submitted +- strip username from package URL on submit (#53939) +- clearer error message when svn co fails +- svn authentication errors are handled, pointing to the wiki page on + configuration +- svn commands will not be shown on error messages, unless using --debug +- repsys uses the wrapper repsys-ssh for svn, to handle signals and also + set BatchMode +- added the configuration option svn-env +- show the error from rpm when fetching the version from a spec fails + +* 1.7 +- dropped all authenticated access support: subversion authentication has + been broken for a long time and the workarounds weren't decent. It will + be back in 1.7.x. +- added configuration option svn-command in the global section, allowing + to replace the default svn command +- force svn+ssh:// URLs to be in BatchMode, in order to not have any + interactivity at all with ssh +- fixed incompatibility with Python-2.4 on urlparse +- fixed emptylog message, which was not being shown when needed +- distributions can be specified by using / in all + commands +- allow submitting many packages at once (#28352) +- package revisions in submit are now specified with name@nnn +- the user can define groups of packages to be submitted in the section + "submit-groups" +- make 'repsys submit' without package name or revision number work again +- added option --distro to submit +- make putsrpm work again +- added subcommand import as an alias to putsrpm +- template: hide the first release when it has only invisible lines +- added initial man page +- allow resorting changelog entries through the config option sort in the + log section +- added rpmlog options: -o to append the old changelog, -p to append the + changelog found in the spec, and -s to resort all changelog entries +- rpmlog, getsrpm, getspec and changed will use the mirror if enabled +- don't hide authors with only the first revision SILENTed (#41117) +- fixed bad url used when using -v in getsrpm +- if REPSYS_CONF is set, /etc/repsys.conf and ~/.repsys/config will not be + readed anymore +- sort the final changelog by enabling the option sort in the log section +- merge the changelog found in the spec by enabling the option merge-spec + in the log section +- changed the built-in template to the current default.chlog +- added option -d to repsys sync, to download the missing source files +- added option -F to repsys ci to set a log message file +- added option --strict to getsrpm to check if the revision provided + matches the package URL; +- changed the default command to build SRPMs to rpmbuild +- added configuration option rpmbuild to the section helper, to define the + command used to build packages +- added boolean configuration option strict-revision in the submit + section, to allow forcing the use of --strict +- added option --list in create-srpm to list the available targets +- make submit -l work on svn+ssh:// targets +- the fix for the unreleased commits problem in the previous release was + wrong, really fixed it +- moved all configuration options that will hardly be changed to + repsys-example.conf; we now have a shorter repsys.conf +- fixed the use of file:/// URLs when using just the package name +- allow using submit with package URLs having usernames +- don't give the wrong message "invalid command 'CMD'" when this is not + the case +- added more help messages in subcommands + +* 1.6.19 +- added complement for SILENT: CLOG, which hides everything that does not + start with this token +- fixed generation of unreleased commits, it was using the previous + markrelease revision as reference +- added option -o to 'co' to disable the use of mirror when checking out + +* 1.6.18 +- added the subcommand "switch" to help with mirrors support +- initialize plugins in create-srpm too +- changelog: perform less svn calls to obtain release number and oldlog +- changelog: show epoch even in the entry not released +- changelog: make default.chlog compatible with cheetah-2 +- make "sync" compatible with rpm-4.4.8 behavior +- "co" don't use mirror when URL is provided +- "ci" don't relocate back to mirrors after commit (should use switch) +- ldapusers: added options ldap-uri and ldap-starttls +- fixed use of __import__, incompatible with python2.4 in plugin support +- fixed bug of wrong paths when using mirrors + +* 1.6.17 +- brought from mdvsys world the sync command +- ldapusers: the configuration format has changed, now it uses python + template strings +- ldapusers: many fixes: better error messages, ldap-port working, results + contain only the fields needed, unbinding after search, filters are + escaped + +* 1.6.16 +- introduced the plugin ldapusers: repsys user data obtained from LDAP; + this plugin is builtin +- added support to plugins, and the hability to wrap configuration sections +- added workaround in the template to ignore empty releases +- added initial support to mirrors, as requested by mrl; it required the + new subcommand "ci" +- changelogs from misc/ will come from HEAD and should be escaped (%%) + +* 1.6.15 +- empty changelog entries are now shown, with a EMPTYLOG tag to allow + rpmlint warn the developer about it +- check (and warn) if a temporary package has already been removed before + trying to remove it + +* 1.6.2b +- make submit pass --define options to create-srpm script +- print error message when create-srpm fails +- make get_srpm return the srpms list +- add upload-srpm support in create-srpm + +* 1.6.2a +- moved revision-offset to [log] section and added a comment + +* 1.6.2 +- reimplemented the option -n for rpmlog, which now uses the svn option + --limit. +- added the option revision-offset, for the Zero Day Revision issue. +- small fix in main repsys help message + +* 1.6.1 +- added option url-map, as an workaround for svn+ssh:// urls problems +- added configuration sectiom "helper" +- added getsrpm-mdk and create-srpm to setup.py +- added option -r to submit + +* 1.6.0 +- improved markrelease command line parsing +- changelogs entries are now groupped by author, and sorted by revision + number +- the changelog now is generated using the Cheetah Template Engine, to + allow quick modifications without spending time reading code and + introducing new bugs +- consequently, was added an option "-T " to rpmlog and getsrpm to + allow choosing the path of the template to be used +- added options noauth=0, and baseurl=None in order to disable the + authentication in some url schemes (http:// and file://) +- replaced some "cl" references to "mdv" diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..acf8138 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,10 @@ +recursive-include RepSys *.py +include RepSys/plugins/*.txt +include repsys repsys.conf MANIFEST.in +include repsys.8 +include README +include CHANGES +include README.LDAP +include repsys-example.conf +include *.chlog +include create-srpm repsys-ssh diff --git a/README b/README new file mode 100644 index 0000000..74aea90 --- /dev/null +++ b/README @@ -0,0 +1,12 @@ +repsys is the tool used to manage RPM packages in a subversion repository. +It is used to create, tag releases, generate .src.rpm, generate changelog, +and request new package releases for build. It mostly acts as a interface +to svn(1) commands and small task scripts run that on the build system side +over ssh(1). + +For more information, see repsys(8) and +http://wiki.mandriva.com/en/Development/Packaging/RepositorySystem + +The discussion on the development of repsys takes place on the +"maintainers" mailing list: +http://wiki.mandriva.com/en/Development/Mailinglists diff --git a/README.LDAP b/README.LDAP new file mode 100644 index 0000000..c22a7fa --- /dev/null +++ b/README.LDAP @@ -0,0 +1,61 @@ +A Repsys plugin for obtaining users from a LDAP server. + +In order to enable the plugin, the user must define the following +options in the [global] section of repsys.conf: + + ldap-uri [required if ldap-server is unset] + the URI of the server, you can refer to more than one server by + adding more URIs separated by spaces:: + + ldap-uri = ldap://ldap.network/ ldaps://backup.network:22389/ + + ldap-server [required if ldap-uri is unset] + the host name of the LDAP server + ldap-port [optional] [default: 389] + the port of the LDAP server + ldap-base [required] + the base DN where the search will be performed + ldap-binddn [optional] [default: empty] + the DN used to bind + ldap-bindpw [optional] [default: empty] + the password used to bind + ldap-starttls [optional] [default: no] + use "yes" or "no" to enable or disable the use of the STARTTLS + LDAP extension + ldap-filterformat [optional] + [default: (&(objectClass=inetOrgPerson)(uid=$username))] + RFC-2254 filter string used in the search of the user entry. + Note that this is a python template string and will have the + user name as parameter. For example: + + ldap-filterformat = (&(objectClass=inetOrgPerson)(uid=$username)) + + Will result in the search filter: + + (&(objectClass=inetOrgPerson)(uid=john)) + + ldap-resultformat [optional] [default: $cn <$mail>] + This is a python template string. This string will be + formatted using one dict object containing the fields + returned in the LDAP search, for example: + + >>> format = Template("$cn <$mail>") + >>> d = search(basedn, filter) + >>> d + {"cn": "John Doe", "mail": "john@mandriva.org", + "uidNumber": "1290", "loginShell": "/bin/bash", + ... many other attributes ... } + >>> value = format.substitute(d) + >>> print value + John Doe + + Note that only the first value of the attributes will be + used. + +When the searched option is not found, it will try in repsys.conf. All +the values found. (including from repsys.conf) will be cached between +each configuration access. + +This plugin requires the package python-ldap. + +For more information, look http://qa.mandriva.com/show_bug.cgi?id=30549 diff --git a/RepSys/ConfigParser.py b/RepSys/ConfigParser.py new file mode 100644 index 0000000..3b4e213 --- /dev/null +++ b/RepSys/ConfigParser.py @@ -0,0 +1,434 @@ +""" +This is a heavily hacked version of ConfigParser to keep the order in +which options and sections are read, and allow multiple options with +the same key. +""" +from __future__ import generators +import string, types +import re + +__all__ = ["NoSectionError","DuplicateSectionError","NoOptionError", + "InterpolationError","InterpolationDepthError","ParsingError", + "MissingSectionHeaderError","ConfigParser", + "MAX_INTERPOLATION_DEPTH"] + +DEFAULTSECT = "DEFAULT" + +MAX_INTERPOLATION_DEPTH = 10 + +# exception classes +class Error(Exception): + def __init__(self, msg=''): + self._msg = msg + Exception.__init__(self, msg) + def __repr__(self): + return self._msg + __str__ = __repr__ + +class NoSectionError(Error): + def __init__(self, section): + Error.__init__(self, 'No section: %s' % section) + self.section = section + +class DuplicateSectionError(Error): + def __init__(self, section): + Error.__init__(self, "Section %s already exists" % section) + self.section = section + +class NoOptionError(Error): + def __init__(self, option, section): + Error.__init__(self, "No option `%s' in section: %s" % + (option, section)) + self.option = option + self.section = section + +class InterpolationError(Error): + def __init__(self, reference, option, section, rawval): + Error.__init__(self, + "Bad value substitution:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\tkey : %s\n" + "\trawval : %s\n" + % (section, option, reference, rawval)) + self.reference = reference + self.option = option + self.section = section + +class InterpolationDepthError(Error): + def __init__(self, option, section, rawval): + Error.__init__(self, + "Value interpolation too deeply recursive:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\trawval : %s\n" + % (section, option, rawval)) + self.option = option + self.section = section + +class ParsingError(Error): + def __init__(self, filename): + Error.__init__(self, 'File contains parsing errors: %s' % filename) + self.filename = filename + self.errors = [] + + def append(self, lineno, line): + self.errors.append((lineno, line)) + self._msg = self._msg + '\n\t[line %2d]: %s' % (lineno, line) + +class MissingSectionHeaderError(ParsingError): + def __init__(self, filename, lineno, line): + Error.__init__( + self, + 'File contains no section headers.\nfile: %s, line: %d\n%s' % + (filename, lineno, line)) + self.filename = filename + self.lineno = lineno + self.line = line + +class ConfigParser: + def __init__(self, defaults=None): + # Options are stored in __sections_list like this: + # [(sectname, [(optname, optval), ...]), ...] + self.__sections_list = [] + self.__sections_dict = {} + if defaults is None: + self.__defaults = {} + else: + self.__defaults = defaults + + def defaults(self): + return self.__defaults + + def sections(self): + return self.__sections_dict.keys() + + def has_section(self, section): + return self.__sections_dict.has_key(section) + + def options(self, section): + self.__sections_dict[section] + try: + opts = self.__sections_dict[section].keys() + except KeyError: + raise NoSectionError(section) + return self.__defaults.keys()+opts + + def read(self, filenames): + if type(filenames) in types.StringTypes: + filenames = [filenames] + for filename in filenames: + try: + fp = open(filename) + except IOError: + continue + self.__read(fp, filename) + fp.close() + + def readfp(self, fp, filename=None): + if filename is None: + try: + filename = fp.name + except AttributeError: + filename = '' + self.__read(fp, filename) + + def set(self, section, option, value): + if self.__sections_dict.has_key(section): + sectdict = self.__sections_dict[section] + sectlist = [] + self.__sections_list.append((section, sectlist)) + elif section == DEFAULTSECT: + sectdict = self.__defaults + sectlist = None + else: + sectdict = {} + self.__sections_dict[section] = sectdict + sectlist = [] + self.__sections_list.append((section, sectlist)) + xform = self.optionxform(option) + sectdict[xform] = value + if sectlist is not None: + sectlist.append([xform, value]) + + def get(self, section, option, raw=0, vars=None): + d = self.__defaults.copy() + try: + d.update(self.__sections_dict[section]) + except KeyError: + if section != DEFAULTSECT: + raise NoSectionError(section) + if vars: + d.update(vars) + option = self.optionxform(option) + try: + rawval = d[option] + except KeyError: + raise NoOptionError(option, section) + if raw: + return rawval + return self.__interpolate(rawval, d) + + def getall(self, section, option, raw=0, vars=None): + option = self.optionxform(option) + values = [] + d = self.__defaults.copy() + if section != DEFAULTSECT: + for sectname, options in self.__sections_list: + if sectname == section: + for optname, value in options: + if optname == option: + values.append(value) + d[optname] = value + if raw: + return values + if vars: + d.update(vars) + for i in len(values): + values[i] = self.__interpolate(values[i], d) + return values + + def walk(self, section, option=None, raw=0, vars=None): + # Build dictionary for interpolation + try: + d = self.__sections_dict[section].copy() + except KeyError: + if section == DEFAULTSECT: + d = {} + else: + raise NoSectionError(section) + d.update(self.__defaults) + if vars: + d.update(vars) + + # Start walking + if option: + option = self.optionxform(option) + if section != DEFAULTSECT: + for sectname, options in self.__sections_list: + if sectname == section: + for optname, value in options: + if not option or optname == option: + if not raw: + value = self.__interpolate(value, d) + yield (optname, value) + + def __interpolate(self, value, vars): + rawval = value + depth = 0 + while depth < 10: + depth = depth + 1 + if value.find("%(") >= 0: + try: + value = value % vars + except KeyError, key: + raise InterpolationError(key, option, section, rawval) + else: + break + if value.find("%(") >= 0: + raise InterpolationDepthError(option, section, rawval) + return value + + def __get(self, section, conv, option): + return conv(self.get(section, option)) + + def getint(self, section, option): + return self.__get(section, string.atoi, option) + + def getfloat(self, section, option): + return self.__get(section, string.atof, option) + + def getboolean(self, section, option): + states = {'1': 1, 'yes': 1, 'true': 1, 'on': 1, + '0': 0, 'no': 0, 'false': 0, 'off': 0} + v = self.get(section, option) + if not states.has_key(v.lower()): + raise ValueError, 'Not a boolean: %s' % v + return states[v.lower()] + + def optionxform(self, optionstr): + #return optionstr.lower() + return optionstr + + def has_option(self, section, option): + """Check for the existence of a given option in a given section.""" + if not section or section == "DEFAULT": + return self.__defaults.has_key(option) + elif not self.has_section(section): + return 0 + else: + option = self.optionxform(option) + return self.__sections_dict[section].has_key(option) + + SECTCRE = re.compile(r'\[(?P
[^]]+)\]') + OPTCRE = re.compile(r'(?P