From ad7fb7807ceaee96521d779993a5e1b28650723f Mon Sep 17 00:00:00 2001 From: Nicolas Vigier Date: Tue, 11 Jan 2011 00:35:59 +0000 Subject: rename repsys to mgarepo, RepSys to MgaRepo, and update docs and examples for Mageia --- BRANCH | 419 ---------------------- MANIFEST.in | 12 +- MgaRepo/ConfigParser.py | 434 +++++++++++++++++++++++ MgaRepo/__init__.py | 19 + MgaRepo/binrepo.py | 395 +++++++++++++++++++++ MgaRepo/cgi/__init__.py | 0 MgaRepo/cgi/soapserver.py | 93 +++++ MgaRepo/cgi/submit.py | 119 +++++++ MgaRepo/cgi/xmlrpcserver.py | 111 ++++++ MgaRepo/cgiutil.py | 53 +++ MgaRepo/command.py | 61 ++++ MgaRepo/commands/__init__.py | 0 MgaRepo/commands/authoremail.py | 37 ++ MgaRepo/commands/changed.py | 41 +++ MgaRepo/commands/ci.py | 35 ++ MgaRepo/commands/co.py | 67 ++++ MgaRepo/commands/create.py | 34 ++ MgaRepo/commands/del.py | 30 ++ MgaRepo/commands/editlog.py | 39 +++ MgaRepo/commands/getspec.py | 38 ++ MgaRepo/commands/getsrpm.py | 100 ++++++ MgaRepo/commands/log.py | 62 ++++ MgaRepo/commands/markrelease.py | 103 ++++++ MgaRepo/commands/patchspec.py | 38 ++ MgaRepo/commands/putsrpm.py | 59 ++++ MgaRepo/commands/rpmlog.py | 68 ++++ MgaRepo/commands/submit.py | 211 +++++++++++ MgaRepo/commands/switch.py | 33 ++ MgaRepo/commands/sync.py | 38 ++ MgaRepo/commands/up.py | 22 ++ MgaRepo/commands/upload.py | 28 ++ MgaRepo/layout.py | 207 +++++++++++ MgaRepo/log.py | 633 +++++++++++++++++++++++++++++++++ MgaRepo/mirror.py | 129 +++++++ MgaRepo/plugins/__init__.py | 27 ++ MgaRepo/plugins/ldapusers.py | 189 ++++++++++ MgaRepo/plugins/sample.py.txt | 14 + MgaRepo/rpmutil.py | 759 ++++++++++++++++++++++++++++++++++++++++ MgaRepo/simplerpm.py | 19 + MgaRepo/svn.py | 430 +++++++++++++++++++++++ MgaRepo/util.py | 141 ++++++++ README | 11 +- README.BINREPO | 360 +++++++++++++++++++ README.LDAP | 6 +- RepSys/ConfigParser.py | 434 ----------------------- RepSys/__init__.py | 19 - RepSys/binrepo.py | 395 --------------------- RepSys/cgi/__init__.py | 0 RepSys/cgi/soapserver.py | 93 ----- RepSys/cgi/submit.py | 119 ------- RepSys/cgi/xmlrpcserver.py | 111 ------ RepSys/cgiutil.py | 53 --- RepSys/command.py | 61 ---- RepSys/commands/__init__.py | 0 RepSys/commands/authoremail.py | 37 -- RepSys/commands/changed.py | 41 --- RepSys/commands/ci.py | 35 -- RepSys/commands/co.py | 67 ---- RepSys/commands/create.py | 34 -- RepSys/commands/del.py | 30 -- RepSys/commands/editlog.py | 39 --- RepSys/commands/getspec.py | 38 -- RepSys/commands/getsrpm.py | 100 ------ RepSys/commands/log.py | 62 ---- RepSys/commands/markrelease.py | 103 ------ RepSys/commands/patchspec.py | 38 -- RepSys/commands/putsrpm.py | 59 ---- RepSys/commands/rpmlog.py | 68 ---- RepSys/commands/submit.py | 211 ----------- RepSys/commands/switch.py | 33 -- RepSys/commands/sync.py | 38 -- RepSys/commands/up.py | 22 -- RepSys/commands/upload.py | 28 -- RepSys/layout.py | 207 ----------- RepSys/log.py | 633 --------------------------------- RepSys/mirror.py | 129 ------- RepSys/plugins/__init__.py | 27 -- RepSys/plugins/ldapusers.py | 189 ---------- RepSys/plugins/sample.py.txt | 14 - RepSys/rpmutil.py | 759 ---------------------------------------- RepSys/simplerpm.py | 19 - RepSys/svn.py | 430 ----------------------- RepSys/util.py | 141 -------- create-srpm | 10 +- default.chlog | 2 +- mgarepo | 93 +++++ mgarepo-example.conf | 74 ++++ mgarepo-ssh | 2 + mgarepo.8 | 219 ++++++++++++ mgarepo.conf | 20 ++ repsys | 93 ----- repsys-example.conf | 74 ---- repsys-ssh | 2 - repsys.8 | 218 ------------ repsys.conf | 20 -- setup.cfg | 2 +- setup.py | 20 +- 97 files changed, 5715 insertions(+), 5774 deletions(-) delete mode 100644 BRANCH create mode 100644 MgaRepo/ConfigParser.py create mode 100644 MgaRepo/__init__.py create mode 100644 MgaRepo/binrepo.py create mode 100644 MgaRepo/cgi/__init__.py create mode 100644 MgaRepo/cgi/soapserver.py create mode 100644 MgaRepo/cgi/submit.py create mode 100644 MgaRepo/cgi/xmlrpcserver.py create mode 100644 MgaRepo/cgiutil.py create mode 100644 MgaRepo/command.py create mode 100644 MgaRepo/commands/__init__.py create mode 100644 MgaRepo/commands/authoremail.py create mode 100644 MgaRepo/commands/changed.py create mode 100644 MgaRepo/commands/ci.py create mode 100644 MgaRepo/commands/co.py create mode 100644 MgaRepo/commands/create.py create mode 100644 MgaRepo/commands/del.py create mode 100644 MgaRepo/commands/editlog.py create mode 100644 MgaRepo/commands/getspec.py create mode 100644 MgaRepo/commands/getsrpm.py create mode 100644 MgaRepo/commands/log.py create mode 100644 MgaRepo/commands/markrelease.py create mode 100644 MgaRepo/commands/patchspec.py create mode 100644 MgaRepo/commands/putsrpm.py create mode 100644 MgaRepo/commands/rpmlog.py create mode 100644 MgaRepo/commands/submit.py create mode 100644 MgaRepo/commands/switch.py create mode 100644 MgaRepo/commands/sync.py create mode 100644 MgaRepo/commands/up.py create mode 100644 MgaRepo/commands/upload.py create mode 100644 MgaRepo/layout.py create mode 100644 MgaRepo/log.py create mode 100644 MgaRepo/mirror.py create mode 100644 MgaRepo/plugins/__init__.py create mode 100644 MgaRepo/plugins/ldapusers.py create mode 100644 MgaRepo/plugins/sample.py.txt create mode 100644 MgaRepo/rpmutil.py create mode 100644 MgaRepo/simplerpm.py create mode 100644 MgaRepo/svn.py create mode 100644 MgaRepo/util.py create mode 100644 README.BINREPO delete mode 100644 RepSys/ConfigParser.py delete mode 100644 RepSys/__init__.py delete mode 100644 RepSys/binrepo.py delete mode 100644 RepSys/cgi/__init__.py delete mode 100644 RepSys/cgi/soapserver.py delete mode 100644 RepSys/cgi/submit.py delete mode 100644 RepSys/cgi/xmlrpcserver.py delete mode 100644 RepSys/cgiutil.py delete mode 100644 RepSys/command.py delete mode 100644 RepSys/commands/__init__.py delete mode 100644 RepSys/commands/authoremail.py delete mode 100644 RepSys/commands/changed.py delete mode 100644 RepSys/commands/ci.py delete mode 100644 RepSys/commands/co.py delete mode 100644 RepSys/commands/create.py delete mode 100644 RepSys/commands/del.py delete mode 100644 RepSys/commands/editlog.py delete mode 100644 RepSys/commands/getspec.py delete mode 100644 RepSys/commands/getsrpm.py delete mode 100644 RepSys/commands/log.py delete mode 100644 RepSys/commands/markrelease.py delete mode 100644 RepSys/commands/patchspec.py delete mode 100644 RepSys/commands/putsrpm.py delete mode 100644 RepSys/commands/rpmlog.py delete mode 100644 RepSys/commands/submit.py delete mode 100644 RepSys/commands/switch.py delete mode 100644 RepSys/commands/sync.py delete mode 100644 RepSys/commands/up.py delete mode 100644 RepSys/commands/upload.py delete mode 100644 RepSys/layout.py delete mode 100644 RepSys/log.py delete mode 100644 RepSys/mirror.py delete mode 100644 RepSys/plugins/__init__.py delete mode 100644 RepSys/plugins/ldapusers.py delete mode 100644 RepSys/plugins/sample.py.txt delete mode 100644 RepSys/rpmutil.py delete mode 100644 RepSys/simplerpm.py delete mode 100644 RepSys/svn.py delete mode 100644 RepSys/util.py create mode 100755 mgarepo create mode 100644 mgarepo-example.conf create mode 100755 mgarepo-ssh create mode 100644 mgarepo.8 create mode 100644 mgarepo.conf delete mode 100755 repsys delete mode 100644 repsys-example.conf delete mode 100755 repsys-ssh delete mode 100644 repsys.8 delete mode 100644 repsys.conf diff --git a/BRANCH b/BRANCH deleted file mode 100644 index ebb87c7..0000000 --- a/BRANCH +++ /dev/null @@ -1,419 +0,0 @@ -================================ -The detached binaries repository -================================ - -.. contents:: - -A brief description -=================== - -Ideally, all binaries from packages sources (ie. all the binary files inside -SOURCES/) will be placed in another subversion repository. This repository -is called "tarballs repository", "binaries repository" or just "binrepo". -It will contain mostly the same directory structure of the main repository, -but instead of having SOURCES and SPECS, it will only have a SOURCES -directory. Every copy/move operation should happen in both repositories. - -In order to allow deceasing binaries from older distributions, each stable -distro will have its own subversion repository for binary files. repsys -knows how to access these binrepos by checking which URL defined in the -"[binrepo]" section of the configuration file matches the path-part of the -repository being accessed. (see open issues) - -The package changelogs will be generated from SVN commit logs in the main -"plaintext" repository ("txtrepo" for short) only. Old changelogs will be -preserved, as even empty revisions are preserved in the binaries-filtering -conversion. - - -Mapping repositories states ---------------------------- - -In order to allow the use of `repsys {getsrpm,co} -r REV`, repsys will have -to use a reference in the text repo which will be used to know in what -state was the binrepo when a binary was uploaded. - -We cannot use direct revision number mapping through properties/files/etc -mainly because we may have multiple binaries repositories, and eventually -they can be filtered for reducing space, thus can't ensure revisions will -survive. Thus another mechanism which relies on dates instead of revisions -numbers is needed. - -When a binary is uploaded to the binrepo, the file `sha1.lst` is updated to -have the files's hash and commited in the main text repo. This file will be -used as the reference when the user uses -r REV on repsys. repsys will -checkout the package in the main text repo with -r REV and then will use -the "Last Changed Date" of `sha1.lst` to checkout the binrepo part. Thus, -`sha1.lst` should be always commited to the main text repository *after* the -corresponding binary files have been commited to the binrepo. Hooks in the -main repository may be used to try to enforce this, by checking if the files -changed in `sha1.lst` are already commited in the corresponding binrepo. - -Computation of `sha1.lst` is unlikely to be an issue: - -- it should not happen too often for any given package -- it takes[0] less than 10s to sha1sum all SOURCES of openoffice.org-3.1-1mdv2010.0.src.rpm -- it probably takes way less than the time to upload the file into the repository -- it can be computed in parallel to the binrepo commit, and probably finish - before that, thus ready by the time `sha1.lst` should be commited -- users don't need to verify the SHA1s "everytime", but the build system - does, thus Repsys can default to not verify and avoid wasting users' time - -The use of `sha1.lst` has the valuable property of tying the state of the main -repository and the binrepo. With it, at getsrpm time of a package -submission we can verify the SHA1 of the SOURCES-bin, and be sure that -either the package will be built with the expected state, or early fail the -build. It also allows for verifying binaries without trusting the binrepo, -which may be useful if we consider using an unversioned plain filesystem -storage in the future (for old distros or whatever), or at "client side", -which maintainers may find useful. - -[0]: In a single core AMD Athlon(tm) 3800+ (2400Mhz) - -Mapping of revisions using SVN properties ------------------------------------------ - -Alternatively to using the above "sha1.lst scheme", the revision mapping -between the main repository and a binrepo could be done using subversion -properties. This could be done by making every commit to binrepos also -cause a corresponding commit in the main text repository to happen, which -would update a property recording the current date. That is, a subversion -property in the main text repository would be kept, such that for any given -main repository revision, the corresponding state of the binrepos is -obtainable (using the registered date). - -This would be "more transparent", as it can be maintened simply by using -subversion hooks, without user intervention. OTOH, as every time the user -commits to a binrepo this would result in a commit in the main repository, -it would require the user to "svn up" the directories from there before -commiting, after every binrepo commit. Also, this might result in a big -number of "bogus" commits to the main repository, which could be seen as log -pollution, and may potentially increase space usage etc.. - -Why a new repository without the tarballs -========================================== - -- the current svn repository is too large, hard to manage -- big binary files (in general, "tarballs") history is of little value in - the distro development, we care much more about our specs, patches, - configurations, etc.; nonetheless, those big files we don't care much for - take the most resources and make backups and restoration in case of - failure very expensive, much more so than the more valuable data -- there is no easy way to strip undesired tarballs without recreating the - whole repository -- fedora and ubuntu have separated repositories, so we must have it too! - -Numbers -------- - -Current repository is +390000 revisions and ~340Gb big, while the bzip2ed -dumps backup for it takes about a bit more than half that size (FIXME: -estimative, can't check in the backup server right now). Current txtrepo -with the same number of revisions is ~180Gb big, takes about 2-3 days to be -imported, while the gzipped full dump backup for it currently takes ~1.2Gb. -Initial binrepo for Cooker (only `current/` packages' branches) took ~28Gb -in disk, gzipped full dump for it takes ~25Gb, took about 5h30m to be -populated from the current in use repository ("oldrepo"). - - -Drawbacks of this layout -========================= - -- (always) everything that changes the single-repository usage increases the chance - of failure and make things more complicated. -- subversion can't be used alone as easily as the current scheme allows -- copying binaries between distro branches may not be "svn-cheap" anymore - (unless they're in the same binrepo) -- ... - - -Open issues -============ - -Multiple binrepos dont allow us to have one permanent URL ---------------------------------------------------------- - -We would have to update the configuration files from all the users in order -to add a new stable repository. spuk suggests to use properties in the main -text repo that would point to the right repository locations. - -How to handle failures when operating on more repositores? ----------------------------------------------------------- - -binrepos should replicate the structure of the main text repo. What we -should do if the markrelease succeeds in the binrepo, but fails in the main -text repo? - -R: Markrelease must be done first in the txtrepo. If it fails there "we're -in trouble" (though currently, we just miss it[0]). When the markrelease is -done in the txtrepo, we can do markrelease in the binrepo using '-r {DATE}', -using the markrelease date in the txtrepo as '{DATE}'. - -[0] We should add transaction support for markrelease. The transaction could -be stored out of the packages SVN (another SVN, a DB, a txt file, etc.), and -would work like: - -0. mark beginning of markrelease, early failing the package build if it fails -1. do markrelease -2. mark sucessful end of markrelease - or mark failed markrelease, so we can replay it later - - -Interesting use cases (first phase) -=================================== - -repsys co 2008.1/mutt ---------------------- - -- repsys checkouts - http://svn.mandriva.com/svn/packages/updates/2008.1/mutt/current to the - mutt directory - -- repsys checkouts - http://svn.mandriva.com/svn/binrepo/updates/2008.1/mutt/current/SOURCES - into mutt/SOURCES-bin - -- creates symlinks for all files found in SOURCES-bin/ into ../SOURCES/ - - (rpm doesn't handle symlinks, this allows us to have explicit links and - proper src.rpm generates by rpmbuild) - -In case the path doesn't exist in the binrepo it will not fail, as we may -have not imported all packages or the repository is not prepared to work on -this model, etc. - -markrelease of a package ------------------------- - -:: - - $ repsys markrelease - -- will copy current/ to releases/VERSION/RELEASE, as usual - -- will copy current/ to releases/, on the binrepo too - -Optionally, markrelease could create revprops indicating which is the -revision of current/ on the binrepo that represents the tarballs that are -being tagged. - - -Use cases to be implemented after the first phase -================================================= - -upgrading to a newer version of the package -------------------------------------------- - -:: - - $ cd bla/SOURCES/ - $ wget http://prdownloads.sourceforge.net/bla/bla-1.6.tar.bz2 - $ repsys add bla-1.6.0.tar.bz2 - -- repsys notices this is a tarball (checking filename and/or file size) - -- repsys will move the file to SOURCES-bin/, create the symlink, and svn-add - it to the working copy - - $ # the user updates the spec - - $ repsys rm SOURCES/bla-1.5.1.tar.bz2 - -- it will remove the symlink and run svn rm on - SOURCES-bin/bla-1.6.0.tar.bz2:: - - $ cd ../ # package top dir - $ repsys ci - -- repsys will commit the new tarball on SOURCES-bin/ and then on the rest - of the working copy - -repsys sync would perform these steps too. - -importing a package -------------------- - - $ repsys putsrpm mypkg.src.rpm - -- repsys will open the src.rpm - -- will look for tarballs inside SOURCES/ and import them to - http://svn.mandriva.com/svn/binrepo/cooker/mypkg/current/SOURCES/ - -- will move the tarballs out of SOURCES and import the remaining files to - http://svn.mandriva.com/svn/packages/cooker/mypkg/current/ - -- will do whatever else putsrpm already does - -TODO -===== - -First phase ------------ - -- upload -- markrelease -- putsrpm -- getsrpm - - -Second phase ------------- - -- up -- sync - -Rejected or postponed ideas -=========================== - -Use of a plain filesystem storage for the tarballs --------------------------------------------------- - -This was planned, then rejected. It becomes too complicated when thinking -about markrelease, and mapping SVN revisions in the main repository to -binaries versions in the "tarballs storage", basically requiring -implementing VCS-like features on top of filesystem. Would also require -implementing another authentication and access scheme. The main feature -would be ease of removing old binaries, which isn't much of a point because -we don't know precisely what and when we want to remove, so may end up not -removing much files anyway. - -Use of a plain unversioned filesystem storage for the tarballs --------------------------------------------------------------- - -Different than the previous one, this would mean not relying at all on -binary files history keeping. Structure could be something simple like:: - - packages/${pkg:0:1}/$pkg/$tarball - -This alternative does not suffice for Cooker, nor for supported distros, for -which we want history. It could, however, at some point be used for "very -old" distros, for which we may have lost interest in keeping *binaries* -history (package history will kept "forever" in the main SVN repository). -Alternatively, "resetting" an SVN binrepo (i.e. recreate the repository) to -contain only the latest tarballs would probably take about the same amount -of space, anyway... - -Open tarballs repository ------------------------- - -This idea is not really rejected. It does not go against splitting txtrepo -and binrepo, but rather complement this idea, where the -open-tarballs-repository would take the place of the binrepo. The txtrepo -would still be used +- the same way. This repository could be used -selectively, for packages where it makes sense, while most packages could be -kept "closed", still as tarballs. - -Use of externals for more seamless Subversion usage ---------------------------------------------------- - -This idea is not discarded, but it just provides easiness. OTOH, it makes -things more complicated: - -- markrelease: externals would have to be updated in order to make it point - to the tagged version in the binrepo, otherwise changes in - current@binrepo would change older releases; -- branching whole distro: even though subversion now supports "relative - externals", we would have to update the URLs for *every* package on the - distro, as the path to reach the binrepo spans the local distribution - directory; -- keeping externals up-to-date (as stated above and below) -- authentication and access control: only markrelease action done by the - build system should be allowed to change externals (so what about importing - new packages?) -- just a convenience, we don't need and shouldn't rely on externals for - running the build system, while most people will use the repositories via - Repsys, so why spend time to implement and keep it? -- "svn co" works transparently, cool, but "svn co -r N" does not, otherwise - every change in the binrepo would require svn:externals to be updated in - the respective package; -- it does not solve the problem of creating and handling symlinks between - SOURCES and SOURCES-bin. - -Keeping svn:externals updated for every package has almost the same cost of -keeping the `sha1.lst` updated, with the difference that in the latter we -would not have to update every package when creating distro branches. - -Use of "external" xdelta to save space on binaries --------------------------------------------------- - -But how? First idea is this could be done by defining a protocol and -assuming repository manipulation with repsys (for ease). Repsys could -xdelta tarballs and add it to SVN with a special filename, then use it when -checking out. Would require a policy/algorithm on when to ditch old whole -binaries, too (i.e. hopefully wouldn't need to be handled manually by the -maintainer). Also, this is something complemental to splitting the -repository, so we may do it later, for binrepos. - - -The Future -========== - -- Open tarballs repositories - - - suited for GIT, maybe multi-VCS - - incremental move - - not everything will be suited for this, must handle all cases or be - optional - -- Xdelta - - -Deployment -========== - -The current repository will be kept around for a while, in readonly state. -Initial binrepos will be populated with the binaries in the `current/` -branches of packages. - -The binrepo mappings config might be kept in a fixed subversion revision -property (revision 0?). - -Rough steps ------------ - -- check for agreement between subversion repository filters for binaries, - and repsys -- upgrade repsys everywhere - - - kenobi - - cluster nodes - - raoh - - titan - -- populate the binrepos for each supported distro, from a specific revision - of oldrepo, and mass commmit the corresponding `sha1.lst` in txtrepo for - every package - - - set svn:date revprop of the `sha1.lst` mass commit to the date of the - oldrepo revision - - before mass commiting the `sha1.lst`, possibly freeze oldrepo, check - for changes to sources after the selected revision, and update the - binrepo as necessary - -- check Secteam scripts, make needed changes to get them ready (non - critical) -- set up the new repositories - - - hook for filtering of disallowed (binary) files in main repository - - binrepos mappings - -- make the new main + binrepos repositories available, but readonly - - - keep new main repository in sync with the old repository with hooks - -- make current repository readonly and enable verification of sha1.lst at - package submission time - -- make sure new main repository and old repository are in sync - - - resync binrepos with the old repository as needed - -- final tests - - - change something - - submit - - etc. - -- make the new repositories writeable - diff --git a/MANIFEST.in b/MANIFEST.in index acf8138..c0c0cfb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,10 @@ -recursive-include RepSys *.py -include RepSys/plugins/*.txt -include repsys repsys.conf MANIFEST.in -include repsys.8 +recursive-include MgaRepo *.py +include MgaRepo/plugins/*.txt +include mgarepo mgarepo.conf MANIFEST.in +include mgarepo.8 include README include CHANGES include README.LDAP -include repsys-example.conf +include mgarepo-example.conf include *.chlog -include create-srpm repsys-ssh +include create-srpm mgarepo-ssh diff --git a/MgaRepo/ConfigParser.py b/MgaRepo/ConfigParser.py new file mode 100644 index 0000000..bac6d36 --- /dev/null +++ b/MgaRepo/ConfigParser.py @@ -0,0 +1,434 @@ +""" +This is a heavily hacked version of ConfigParser to keep the order in +which options and sections are read, and allow multiple options with +the same key. +""" +from __future__ import generators +import string, types +import re + +__all__ = ["NoSectionError","DuplicateSectionError","NoOptionError", + "InterpolationError","InterpolationDepthError","ParsingError", + "MissingSectionHeaderError","ConfigParser", + "MAX_INTERPOLATION_DEPTH"] + +DEFAULTSECT = "DEFAULT" + +MAX_INTERPOLATION_DEPTH = 10 + +# exception classes +class Error(Exception): + def __init__(self, msg=''): + self._msg = msg + Exception.__init__(self, msg) + def __repr__(self): + return self._msg + __str__ = __repr__ + +class NoSectionError(Error): + def __init__(self, section): + Error.__init__(self, 'No section: %s' % section) + self.section = section + +class DuplicateSectionError(Error): + def __init__(self, section): + Error.__init__(self, "Section %s already exists" % section) + self.section = section + +class NoOptionError(Error): + def __init__(self, option, section): + Error.__init__(self, "No option `%s' in section: %s" % + (option, section)) + self.option = option + self.section = section + +class InterpolationError(Error): + def __init__(self, reference, option, section, rawval): + Error.__init__(self, + "Bad value substitution:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\tkey : %s\n" + "\trawval : %s\n" + % (section, option, reference, rawval)) + self.reference = reference + self.option = option + self.section = section + +class InterpolationDepthError(Error): + def __init__(self, option, section, rawval): + Error.__init__(self, + "Value interpolation too deeply recursive:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\trawval : %s\n" + % (section, option, rawval)) + self.option = option + self.section = section + +class ParsingError(Error): + def __init__(self, filename): + Error.__init__(self, 'File contains parsing errors: %s' % filename) + self.filename = filename + self.errors = [] + + def append(self, lineno, line): + self.errors.append((lineno, line)) + self._msg = self._msg + '\n\t[line %2d]: %s' % (lineno, line) + +class MissingSectionHeaderError(ParsingError): + def __init__(self, filename, lineno, line): + Error.__init__( + self, + 'File contains no section headers.\nfile: %s, line: %d\n%s' % + (filename, lineno, line)) + self.filename = filename + self.lineno = lineno + self.line = line + +class ConfigParser: + def __init__(self, defaults=None): + # Options are stored in __sections_list like this: + # [(sectname, [(optname, optval), ...]), ...] + self.__sections_list = [] + self.__sections_dict = {} + if defaults is None: + self.__defaults = {} + else: + self.__defaults = defaults + + def defaults(self): + return self.__defaults + + def sections(self): + return self.__sections_dict.keys() + + def has_section(self, section): + return self.__sections_dict.has_key(section) + + def options(self, section): + self.__sections_dict[section] + try: + opts = self.__sections_dict[section].keys() + except KeyError: + raise NoSectionError(section) + return self.__defaults.keys()+opts + + def read(self, filenames): + if type(filenames) in types.StringTypes: + filenames = [filenames] + for filename in filenames: + try: + fp = open(filename) + except IOError: + continue + self.__read(fp, filename) + fp.close() + + def readfp(self, fp, filename=None): + if filename is None: + try: + filename = fp.name + except AttributeError: + filename = '' + self.__read(fp, filename) + + def set(self, section, option, value): + if self.__sections_dict.has_key(section): + sectdict = self.__sections_dict[section] + sectlist = [] + self.__sections_list.append((section, sectlist)) + elif section == DEFAULTSECT: + sectdict = self.__defaults + sectlist = None + else: + sectdict = {} + self.__sections_dict[section] = sectdict + sectlist = [] + self.__sections_list.append((section, sectlist)) + xform = self.optionxform(option) + sectdict[xform] = value + if sectlist is not None: + sectlist.append([xform, value]) + + def get(self, section, option, raw=0, vars=None): + d = self.__defaults.copy() + try: + d.update(self.__sections_dict[section]) + except KeyError: + if section != DEFAULTSECT: + raise NoSectionError(section) + if vars: + d.update(vars) + option = self.optionxform(option) + try: + rawval = d[option] + except KeyError: + raise NoOptionError(option, section) + if raw: + return rawval + return self.__interpolate(rawval, d) + + def getall(self, section, option, raw=0, vars=None): + option = self.optionxform(option) + values = [] + d = self.__defaults.copy() + if section != DEFAULTSECT: + for sectname, options in self.__sections_list: + if sectname == section: + for optname, value in options: + if optname == option: + values.append(value) + d[optname] = value + if raw: + return values + if vars: + d.update(vars) + for i in len(values): + values[i] = self.__interpolate(values[i], d) + return values + + def walk(self, section, option=None, raw=0, vars=None): + # Build dictionary for interpolation + try: + d = self.__sections_dict[section].copy() + except KeyError: + if section == DEFAULTSECT: + d = {} + else: + raise NoSectionError(section) + d.update(self.__defaults) + if vars: + d.update(vars) + + # Start walking + if option: + option = self.optionxform(option) + if section != DEFAULTSECT: + for sectname, options in self.__sections_list: + if sectname == section: + for optname, value in options: + if not option or optname == option: + if not raw: + value = self.__interpolate(value, d) + yield (optname, value) + + def __interpolate(self, value, vars): + rawval = value + depth = 0 + while depth < 10: + depth = depth + 1 + if value.find("%(") >= 0: + try: + value = value % vars + except KeyError, key: + raise InterpolationError(key, option, section, rawval) + else: + break + if value.find("%(") >= 0: + raise InterpolationDepthError(option, section, rawval) + return value + + def __get(self, section, conv, option): + return conv(self.get(section, option)) + + def getint(self, section, option): + return self.__get(section, string.atoi, option) + + def getfloat(self, section, option): + return self.__get(section, string.atof, option) + + def getboolean(self, section, option): + states = {'1': 1, 'yes': 1, 'true': 1, 'on': 1, + '0': 0, 'no': 0, 'false': 0, 'off': 0} + v = self.get(section, option) + if not states.has_key(v.lower()): + raise ValueError, 'Not a boolean: %s' % v + return states[v.lower()] + + def optionxform(self, optionstr): + #return optionstr.lower() + return optionstr + + def has_option(self, section, option): + """Check for the existence of a given option in a given section.""" + if not section or section == "DEFAULT": + return self.__defaults.has_key(option) + elif not self.has_section(section): + return 0 + else: + option = self.optionxform(option) + return self.__sections_dict[section].has_key(option) + + SECTCRE = re.compile(r'\[(?P
[^]]+)\]') + OPTCRE = re.compile(r'(?P