multiprocessing for clean-spec and remove py_requires

1 files changed, 98 insertions, 66 deletions

diff --git a/mgagnome b/mgagnome
index 381ea15..56671de 100755
--- a/mgagnome
+++ b/mgagnome
@@ -384,11 +384,15 @@ class SpecFile(object):
         re_rm_buildroot = r'^(?:\[[^\n\]\[]+\][ \t]+\&\&[ \t]+)?(?:rm|\%__rm|\%\{__rm\}) *(?:-rf|-fr|-r) *(?:\%buildroot|"\%\{buildroot\}"|\%\{buildroot\}|\$RPM_BUILDROOT|\$RPM_BUILD_ROOT|\$\{RPM_BUILD_ROOT\}|\$RPM_BUILD_DIR)/? *\n'
         re_clean_spec = [
             # remove %defattr
-            ('remove defattr', re.compile(r'(?P<keeppre>^\%files[^\n]*\n)^\%defattr\s*\(-, *root, *root(?:, *-)?\)\s*\n', re.MULTILINE + re.IGNORECASE)),
-            ('remove cleaning buildroot in install', re.compile(r'(?P<keeppre>^\%install[^\n]*\n)' + re_rm_buildroot + r'\n?', re.MULTILINE + re.IGNORECASE)),
-            ('remove clean section', re.compile(r'^\%clean[^\n]*\n(?:' + re_rm_buildroot + r')?\n*(?P<keeppost>^(?:\%files|\%post|\%pre|\%trigger|\%check|\%_font_pkg|$(?!.|\n)))', re.MULTILINE + re.IGNORECASE)),
+            ('remove defattr', re.compile(r'(?P<keeppre>^\%files(?:[ \t]+[^\n]*)?\n)^\%defattr\s*\(-, *root, *root(?:, *-)?\)\s*\n', re.MULTILINE + re.IGNORECASE)),
+            ('remove cleaning buildroot in install', re.compile(r'(?P<keeppre>^\%install(?:[ \t]+[^\n]*)?\n)' + re_rm_buildroot + r'\n?', re.MULTILINE + re.IGNORECASE)),
+            ('remove clean section', re.compile(r'^\%clean[ \t]*\n(?:' + re_rm_buildroot + r')?\n*(?P<keeppost>^(?:\%files|\%post|\%pre|\%trigger|\%check|\%_font_pkg|$(?!.|\n)))', re.MULTILINE + re.IGNORECASE)),
             ('remove buildroot definition', re.compile(r'^BuildRoot:[^\n]+\n', re.MULTILINE + re.IGNORECASE)),
         ]
+        re_convert_br = [
+            ('remove py_requires', ('python',), re.compile(r'^\%py_requires[ \t]*\n', re.MULTILINE)),
+            ('remove py_requires -d', ('python', 'python-devel'), re.compile(r'^\%py_requires[ \t]+-d[ \t]*\n', re.MULTILINE)),
+        ]
 
         made_changes = False
         with open(self.path, "rw") as f:
@@ -402,6 +406,29 @@ class SpecFile(object):
                     made_changes = True
                     self._changes.append('SILENT %s' % reason)
 
+            # Convert %py_requires and %py_requires -d
+            #  - first figure out how a buildrequire is usually defined
+            mo = self.re_update_br.search(data)
+            br_pre = mo.group('pre') if mo and mo.group('pre') else "BuildRequires:\t"
+            print br_pre
+
+            for reason, new_brs, regexp in re_convert_br:
+                mo = regexp.search(data)
+                if mo:
+                    # Don't add multiple buildrequires
+                    change_to = ""
+                    brs_in_file = set()
+                    for mo2 in self.re_update_br.finditer(data):
+                        if mo2.group('br') in new_brs:
+                            brs_in_file.add(mo2.group('br'))
+
+                    for br in set(new_brs) - brs_in_file:
+                        change_to += ''.join((br_pre, br, "\n"))
+                    data, nr = regexp.subn(change_to, data)
+                    if nr:
+                        made_changes = True
+                        self._changes.append('SILENT %s' % reason)
+
             # Convert:
             #   %define name SOMETHING
             #   name: %{name}
@@ -1203,31 +1230,25 @@ def cmd_check_prep(options, parser):
     s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % options.package), module=options.package)
     s.check_and_update_patches()
 
-def cmd_clean_spec(options, parser):
+def cmd_clean_spec_multi(args):
+    options, package = args
     # Directories packages are located in
     root = os.path.expanduser(Downstream.PKGROOT)
-    if options.all:
-        packages = Downstream().packages
-    else:
-        packages = options.package if len(options.package) else (l[0] for l in join_streams())
 
-    import pprint
-
-    for package in packages:
-        print package
-        cwd = os.path.join(root, package)
+    print package
+    cwd = os.path.join(root, package)
 
-        s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package), module=package)
+    s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package), module=package)
 
-        try:
-            if not s.ensure_no_local_changes():
-                continue
-        except subprocess.CalledProcessError:
-            # Package was probably not checked out or something
-            print >>sys.stderr, "ERROR: cannot clean spec file for %s" % package
-            continue
+    try:
+        if not s.ensure_no_local_changes():
+            return False
+    except subprocess.CalledProcessError:
+        # Package was probably not checked out or something
+        print >>sys.stderr, "ERROR: cannot clean spec file for %s" % package
+        return False
 
-        made_changes=False
+    made_changes=False
 
 #        with open(s.path, "r") as f:
 #            data = f.read()
@@ -1237,56 +1258,67 @@ def cmd_clean_spec(options, parser):
 #                for match in s.re_br_part.findall(mo.group('unsplit')):
 #                        print "d", match
 
-        changes = {}
-
-        # Convert -devel requires into pkgconfig requires
-        if options.pkgconfig:
-            br = s.buildrequires
-            br_old = [r for r in br.keys() if r.endswith('-devel')]
-            no_alt = set()
-            no_change = {}
-            for req in br_old:
-                provides = Downstream.alternative_provides(req)
-                provides_alt = [clean_pkgconfig_prov(prov) for prov in provides if prov.startswith('pkgconfig(')]
-                change_to = None
-                if len(provides_alt) == 1:
-                    change_to = provides_alt[0]
-                elif len(provides_alt):
-                    basereq = req[:-len('-devel')]
-                    check_for = ['pkgconfig(%s)' % basereq]
-                    if basereq.startswith('lib'):
-                        check_for.append('pkgconfig(%s)' % basereq[len('lib'):])
-                    else:
-                        check_for.append('pkgconfig(lib%s)' % basereq)
-                    for check in check_for:
-                        if check in provides_alt:
-                            change_to = check
-                            break
+    changes = {}
+
+    # Convert -devel requires into pkgconfig requires
+    if options.pkgconfig:
+        br = s.buildrequires
+        br_old = [r for r in br.keys() if r.endswith('-devel')]
+        no_alt = set()
+        no_change = {}
+        for req in br_old:
+            provides = Downstream.alternative_provides(req)
+            provides_alt = [clean_pkgconfig_prov(prov) for prov in provides if prov.startswith('pkgconfig(')]
+            change_to = None
+            if len(provides_alt) == 1:
+                change_to = provides_alt[0]
+            elif len(provides_alt):
+                basereq = req[:-len('-devel')]
+                check_for = ['pkgconfig(%s)' % basereq]
+                if basereq.startswith('lib'):
+                    check_for.append('pkgconfig(%s)' % basereq[len('lib'):])
+                else:
+                    check_for.append('pkgconfig(lib%s)' % basereq)
+                for check in check_for:
+                    if check in provides_alt:
+                        change_to = check
+                        break
 
 
-                if len(provides_alt):
-                    if change_to is None: no_change[req] = provides_alt
-                else:
-                    no_alt.add(req)
+            if len(provides_alt):
+                if change_to is None: no_change[req] = provides_alt
+            else:
+                no_alt.add(req)
 
-                if change_to is not None: changes[req] = change_to
+            if change_to is not None: changes[req] = change_to
 
-            if not options.doit:
-                if changes: pprint.pprint(changes)
-                if no_alt: print "WARNING: no alternatives found for: %s" % ", ".join(sorted(no_alt))
-                if no_change: pprint.pprint(no_change)
+        if not options.doit:
+            if changes: pprint.pprint(changes)
+            if no_alt: print "WARNING: no alternatives found for: %s" % ", ".join(sorted(no_alt))
+            if no_change: pprint.pprint(no_change)
 
-        if s.update_br(changes, change_description='convert -devel buildrequires into pkgconfig'): made_changes=True
-        if s.clean_spec(): made_changes=True
+    if s.update_br(changes, change_description='convert -devel buildrequires into pkgconfig'): made_changes=True
+    if s.clean_spec(): made_changes=True
 
-        # If we made it this far, checkin the changes
-        if made_changes:
-            if options.doit:
-                subprocess.check_call(['mgarepo', 'ci', '-m', s.changes], cwd=cwd)
-            else:
-                # show the diff and undo all changes
-                print s.changes
-                s.ensure_no_local_changes(force=True)
+    # If we made it this far, checkin the changes
+    if made_changes:
+        if options.doit:
+            subprocess.check_call(['mgarepo', 'ci', '-m', s.changes], cwd=cwd)
+        else:
+            # show the diff and undo all changes
+            print s.changes
+            s.ensure_no_local_changes(force=True)
+
+def cmd_clean_spec(options, parser):
+    if options.all:
+        packages = Downstream().packages
+    else:
+        packages = options.package if len(options.package) else (l[0] for l in join_streams())
+
+    import pprint
+
+    p = multiprocessing.Pool(5)
+    p.map(cmd_clean_spec_multi, ((options, package) for package in packages))
 
 def cmd_package_new_version(options, parser):
     # Determine the package name