From b8f9d03937492a97d84f31ce183c4f76b56ca504 Mon Sep 17 00:00:00 2001 From: Olav Vitters Date: Sun, 6 Jul 2014 17:20:36 +0200 Subject: multiprocessing for clean-spec and remove py_requires --- mgagnome | 164 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 98 insertions(+), 66 deletions(-) diff --git a/mgagnome b/mgagnome index 381ea15..56671de 100755 --- a/mgagnome +++ b/mgagnome @@ -384,11 +384,15 @@ class SpecFile(object): re_rm_buildroot = r'^(?:\[[^\n\]\[]+\][ \t]+\&\&[ \t]+)?(?:rm|\%__rm|\%\{__rm\}) *(?:-rf|-fr|-r) *(?:\%buildroot|"\%\{buildroot\}"|\%\{buildroot\}|\$RPM_BUILDROOT|\$RPM_BUILD_ROOT|\$\{RPM_BUILD_ROOT\}|\$RPM_BUILD_DIR)/? *\n' re_clean_spec = [ # remove %defattr - ('remove defattr', re.compile(r'(?P^\%files[^\n]*\n)^\%defattr\s*\(-, *root, *root(?:, *-)?\)\s*\n', re.MULTILINE + re.IGNORECASE)), - ('remove cleaning buildroot in install', re.compile(r'(?P^\%install[^\n]*\n)' + re_rm_buildroot + r'\n?', re.MULTILINE + re.IGNORECASE)), - ('remove clean section', re.compile(r'^\%clean[^\n]*\n(?:' + re_rm_buildroot + r')?\n*(?P^(?:\%files|\%post|\%pre|\%trigger|\%check|\%_font_pkg|$(?!.|\n)))', re.MULTILINE + re.IGNORECASE)), + ('remove defattr', re.compile(r'(?P^\%files(?:[ \t]+[^\n]*)?\n)^\%defattr\s*\(-, *root, *root(?:, *-)?\)\s*\n', re.MULTILINE + re.IGNORECASE)), + ('remove cleaning buildroot in install', re.compile(r'(?P^\%install(?:[ \t]+[^\n]*)?\n)' + re_rm_buildroot + r'\n?', re.MULTILINE + re.IGNORECASE)), + ('remove clean section', re.compile(r'^\%clean[ \t]*\n(?:' + re_rm_buildroot + r')?\n*(?P^(?:\%files|\%post|\%pre|\%trigger|\%check|\%_font_pkg|$(?!.|\n)))', re.MULTILINE + re.IGNORECASE)), ('remove buildroot definition', re.compile(r'^BuildRoot:[^\n]+\n', re.MULTILINE + re.IGNORECASE)), ] + re_convert_br = [ + ('remove py_requires', ('python',), re.compile(r'^\%py_requires[ \t]*\n', re.MULTILINE)), + ('remove py_requires -d', ('python', 'python-devel'), re.compile(r'^\%py_requires[ \t]+-d[ \t]*\n', re.MULTILINE)), + ] made_changes = False with open(self.path, "rw") as f: @@ -402,6 +406,29 @@ class SpecFile(object): made_changes = True self._changes.append('SILENT %s' % reason) + # Convert %py_requires and %py_requires -d + # - first figure out how a buildrequire is usually defined + mo = self.re_update_br.search(data) + br_pre = mo.group('pre') if mo and mo.group('pre') else "BuildRequires:\t" + print br_pre + + for reason, new_brs, regexp in re_convert_br: + mo = regexp.search(data) + if mo: + # Don't add multiple buildrequires + change_to = "" + brs_in_file = set() + for mo2 in self.re_update_br.finditer(data): + if mo2.group('br') in new_brs: + brs_in_file.add(mo2.group('br')) + + for br in set(new_brs) - brs_in_file: + change_to += ''.join((br_pre, br, "\n")) + data, nr = regexp.subn(change_to, data) + if nr: + made_changes = True + self._changes.append('SILENT %s' % reason) + # Convert: # %define name SOMETHING # name: %{name} @@ -1203,31 +1230,25 @@ def cmd_check_prep(options, parser): s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % options.package), module=options.package) s.check_and_update_patches() -def cmd_clean_spec(options, parser): +def cmd_clean_spec_multi(args): + options, package = args # Directories packages are located in root = os.path.expanduser(Downstream.PKGROOT) - if options.all: - packages = Downstream().packages - else: - packages = options.package if len(options.package) else (l[0] for l in join_streams()) - import pprint - - for package in packages: - print package - cwd = os.path.join(root, package) + print package + cwd = os.path.join(root, package) - s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package), module=package) + s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package), module=package) - try: - if not s.ensure_no_local_changes(): - continue - except subprocess.CalledProcessError: - # Package was probably not checked out or something - print >>sys.stderr, "ERROR: cannot clean spec file for %s" % package - continue + try: + if not s.ensure_no_local_changes(): + return False + except subprocess.CalledProcessError: + # Package was probably not checked out or something + print >>sys.stderr, "ERROR: cannot clean spec file for %s" % package + return False - made_changes=False + made_changes=False # with open(s.path, "r") as f: # data = f.read() @@ -1237,56 +1258,67 @@ def cmd_clean_spec(options, parser): # for match in s.re_br_part.findall(mo.group('unsplit')): # print "d", match - changes = {} - - # Convert -devel requires into pkgconfig requires - if options.pkgconfig: - br = s.buildrequires - br_old = [r for r in br.keys() if r.endswith('-devel')] - no_alt = set() - no_change = {} - for req in br_old: - provides = Downstream.alternative_provides(req) - provides_alt = [clean_pkgconfig_prov(prov) for prov in provides if prov.startswith('pkgconfig(')] - change_to = None - if len(provides_alt) == 1: - change_to = provides_alt[0] - elif len(provides_alt): - basereq = req[:-len('-devel')] - check_for = ['pkgconfig(%s)' % basereq] - if basereq.startswith('lib'): - check_for.append('pkgconfig(%s)' % basereq[len('lib'):]) - else: - check_for.append('pkgconfig(lib%s)' % basereq) - for check in check_for: - if check in provides_alt: - change_to = check - break + changes = {} + + # Convert -devel requires into pkgconfig requires + if options.pkgconfig: + br = s.buildrequires + br_old = [r for r in br.keys() if r.endswith('-devel')] + no_alt = set() + no_change = {} + for req in br_old: + provides = Downstream.alternative_provides(req) + provides_alt = [clean_pkgconfig_prov(prov) for prov in provides if prov.startswith('pkgconfig(')] + change_to = None + if len(provides_alt) == 1: + change_to = provides_alt[0] + elif len(provides_alt): + basereq = req[:-len('-devel')] + check_for = ['pkgconfig(%s)' % basereq] + if basereq.startswith('lib'): + check_for.append('pkgconfig(%s)' % basereq[len('lib'):]) + else: + check_for.append('pkgconfig(lib%s)' % basereq) + for check in check_for: + if check in provides_alt: + change_to = check + break - if len(provides_alt): - if change_to is None: no_change[req] = provides_alt - else: - no_alt.add(req) + if len(provides_alt): + if change_to is None: no_change[req] = provides_alt + else: + no_alt.add(req) - if change_to is not None: changes[req] = change_to + if change_to is not None: changes[req] = change_to - if not options.doit: - if changes: pprint.pprint(changes) - if no_alt: print "WARNING: no alternatives found for: %s" % ", ".join(sorted(no_alt)) - if no_change: pprint.pprint(no_change) + if not options.doit: + if changes: pprint.pprint(changes) + if no_alt: print "WARNING: no alternatives found for: %s" % ", ".join(sorted(no_alt)) + if no_change: pprint.pprint(no_change) - if s.update_br(changes, change_description='convert -devel buildrequires into pkgconfig'): made_changes=True - if s.clean_spec(): made_changes=True + if s.update_br(changes, change_description='convert -devel buildrequires into pkgconfig'): made_changes=True + if s.clean_spec(): made_changes=True - # If we made it this far, checkin the changes - if made_changes: - if options.doit: - subprocess.check_call(['mgarepo', 'ci', '-m', s.changes], cwd=cwd) - else: - # show the diff and undo all changes - print s.changes - s.ensure_no_local_changes(force=True) + # If we made it this far, checkin the changes + if made_changes: + if options.doit: + subprocess.check_call(['mgarepo', 'ci', '-m', s.changes], cwd=cwd) + else: + # show the diff and undo all changes + print s.changes + s.ensure_no_local_changes(force=True) + +def cmd_clean_spec(options, parser): + if options.all: + packages = Downstream().packages + else: + packages = options.package if len(options.package) else (l[0] for l in join_streams()) + + import pprint + + p = multiprocessing.Pool(5) + p.map(cmd_clean_spec_multi, ((options, package) for package in packages)) def cmd_package_new_version(options, parser): # Determine the package name -- cgit v1.2.1