From d2cfcf139fc8f9f874e89c6e30c96a52ba4888a3 Mon Sep 17 00:00:00 2001 From: Olav Vitters Date: Mon, 4 May 2020 17:02:59 +0200 Subject: Introduce a Package class Move various functions from the Downstream class to this newly created Package class. To avoid any confusion, ensure any variables referring to a package name are called package_name instead of just "package". --- mgagnome | 296 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 157 insertions(+), 139 deletions(-) diff --git a/mgagnome b/mgagnome index bc333a6..23560a9 100755 --- a/mgagnome +++ b/mgagnome @@ -1276,6 +1276,11 @@ class Downstream(): """Return all downstream package names""" return sorted(self._packages) + @classmethod + def package(cls, package): + """Return an initialized package""" + return Package(package, cls) + _provide_to_alternate = {} @classmethod def alternative_provides(cls, search_for): @@ -1347,69 +1352,6 @@ class Downstream(): return (set(), set()) - @classmethod - def package_path(cls, package): - """Provide the local checkout path for a given package - - Package might not be checked out yet!""" - return os.path.join(os.path.expanduser(Downstream.PKGROOT), package) - - @classmethod - def package_spec(cls, package): - """Return the SpecFile for a given package""" - path = cls.package_path(package) - - return SpecFile(os.path.join(path, "SPECS", "%s.spec" % package), module=package) - - - @classmethod - @retry(subprocess.CalledProcessError) - def checkout(cls, package, cwd=None, spec_only=False): - """Check out a package from the repository""" - if cwd is None: - cwd = os.path.expanduser(cls.PKGROOT) - - cmd = ['mgarepo', 'co'] - if cls.DISTRO: - cmd.extend(('-d', cls.DISTRO)) - if spec_only: - cmd.append('-s') - cmd.append(package) - return subprocess.check_call(cmd, stdin=subprocess.DEVNULL, cwd=cwd) - - @classmethod - @retry(subprocess.CalledProcessError) - def checkin(cls, package, changes, cwd=None): - """Check in changes to the repository""" - - if cwd is None: - cwd = cls.package_path(package) - - cmd = ['mgarepo', 'ci', '-m', changes] - return subprocess.check_call(cmd, cwd=cwd) - - @classmethod - def submit(cls, package, cwd=None): - """Submit a package to the buildsystem - - If a specific distro release is chosen, the section will be set to - core/updates_testing.""" - if cwd is None: - cwd = cls.package_path(package) - - cmd = ['mgarepo', 'submit'] - if cls.DISTRO: - cmd.extend(('--define', 'section=core/updates_testing', '-t', cls.DISTRO)) - elif cls.SECTION: - cmd.extend(('--define', 'section={0}'.format(cls.SECTION))) - - # retry submission various times, could be that some dependencies are being built at the same time - @retry(subprocess.CalledProcessError, tries=10, delay=300, backoff=1.5) - def _submit(): - subprocess.check_call(cmd, cwd=cwd) - _submit() - - def get_downstream_from_upstream(self, upstream, version): """Provide the downstream package(s) for a given upstream module name and version @@ -1425,21 +1367,22 @@ class Downstream(): return list(self.tarballs[upstream].keys()) packages = {} - for package in list(self.tarballs[upstream].keys()): + for package_name in list(self.tarballs[upstream].keys()): + package = self.package(package_name) # Checkout package to ensure the checkout reflects the latest changes try: - self.checkout(package) + package.checkout() except subprocess.CalledProcessError: - raise ValueError("Multiple packages found and cannot checkout %s" % package) + raise ValueError("Multiple packages found and cannot checkout %s" % package_name) # Determine version from spec file try: - packages[package] = self.package_spec(package).version + packages[package_name] = package.spec.version except subprocess.CalledProcessError: - raise ValueError("Multiple packages found and cannot determine version of %s" % package) + raise ValueError("Multiple packages found and cannot determine version of %s" % package_name) # Return all packages reflecting the current version - matches = [package for package in packages if packages[package] == version] + matches = [package_name for package_name in packages if packages[package_name] == version] if matches: return matches @@ -1460,13 +1403,82 @@ class Downstream(): ) # - now really get the right packages - matches = [package for package in packages if packages[package] == latest_version] + matches = [package for package_name in packages if packages[package_name] == latest_version] if matches: return matches # Give up raise ValueError("Multiple packages found and cannot determine package for version %s" % version) +class Package: + """Represents a downstream package""" + + def __init__(self, packagename, downstreamclass): + self.name = packagename + self._downstream = downstreamclass + + @property + def path(self): + """Provide the local checkout path for a given package + + Package might not be checked out yet!""" + return os.path.join(os.path.expanduser(self._downstream.PKGROOT), self.name) + + @cached_property + def spec(self): + """Return the SpecFile for a given package""" + path = self.path + + return SpecFile(os.path.join(path, "SPECS", "%s.spec" % self.name), module=self.name) + + + @retry(subprocess.CalledProcessError) + def checkout(self, cwd=None, spec_only=False): + """Check out a package from the repository""" + downstream = self._downstream + if cwd is None: + cwd = os.path.expanduser(downstream.PKGROOT) + + cmd = ['mgarepo', 'co'] + if downstream.DISTRO: + cmd.extend(('-d', downstream.DISTRO)) + if spec_only: + downstream.append('-s') + downstream.append(self.name) + return subprocess.check_call(cmd, stdin=subprocess.DEVNULL, cwd=cwd) + + @retry(subprocess.CalledProcessError) + def checkin(self, changes, cwd=None): + """Check in changes to the repository""" + + if cwd is None: + cwd = self.path + + cmd = ['mgarepo', 'ci', '-m', changes] + return subprocess.check_call(cmd, cwd=cwd) + + def submit(self, cwd=None): + """Submit a package to the buildsystem + + If a specific distro release is chosen, the section will be set to + core/updates_testing.""" + downstream = self._downstream + if cwd is None: + cwd = self.path + + cmd = ['mgarepo', 'submit'] + if downstream.DISTRO: + cmd.extend(('--define', 'section=core/updates_testing', '-t', downstream.DISTRO)) + elif downstream.SECTION: + cmd.extend(('--define', 'section={0}'.format(downstream.SECTION))) + + # retry submission various times, could be that some dependencies are being built at the same time + @retry(subprocess.CalledProcessError, tries=10, delay=300, backoff=1.5) + def _submit(): + subprocess.check_call(cmd, cwd=cwd) + _submit() + + def write_file(path, data): """Write to a file by creating a temporary file and renaming that to the new file""" @@ -1477,27 +1489,27 @@ def write_file(path, data): os.rename(fdst.name, path) def _cmd_checkout_multi(args): - package, what_to_print, options = args + package_name, what_to_print, options = args print(what_to_print) try: - Downstream.checkout(package, spec_only=options.spec_only) + Downstream.package(package_name).checkout(spec_only=options.spec_only) except subprocess.CalledProcessError: pass def cmd_checkout(options): """Check out various packages in parallel""" if options.all: - packages = ((package, package, options) for package in Downstream().packages) + packages = ((package_name, package_name, options) for package_name in Downstream().packages) elif options.package: - packages = ((package, package, options) for package in options.package) + packages = ((package_name, package_name, options) for package_name in options.package) else: packages = ((l[0], "%s => %s" % (l[0], l[1]), options) for l in sorted(join_streams(auto_update=False))) if options.debug: - for package in packages: - _cmd_checkout_multi(package) + for package_name in packages: + _cmd_checkout_multi(package_name) else: with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor: executor.map(_cmd_checkout_multi, packages) @@ -1516,36 +1528,37 @@ def join_streams(show_version=False, only_diff_version=False, auto_update=True): matches = upstream & set(downstream.tarballs.keys()) for module in matches: - for package in list(downstream.tarballs[module].keys()): - package_version = downstream.tarballs[module][package] + for package_name in list(downstream.tarballs[module].keys()): + package_version = downstream.tarballs[module][package_name] spec_version = None - cwd = Downstream.package_path(package) + package = Downstream.package(package) + cwd = package.path if show_version or only_diff_version: # ensure package is checked out if not os.path.exists(cwd): try: - downstream.checkout(package) + package.checkout() except subprocess.CalledProcessError: # XXX - ignoring packages which cannot be checked out continue try: - spec_version = Downstream.package_spec(package).version + spec_version = package.spec.version except subprocess.CalledProcessError: spec_version = 'N/A' # in case upstream version is newer, update checkout if auto_update and package_version != spec_version and version_cmp(package_version, spec_version) == 1: try: - downstream.checkout(package) - spec_version = downstream.package_spec(package).version + package.checkout() + spec_version = package.spec.version except subprocess.CalledProcessError: pass if only_diff_version and package_version == spec_version: continue - yield (package, module, package_version, spec_version, downstream.files[package]) + yield (package_name, module, package_version, spec_version, downstream.files[package_name]) def cmd_group_owner(options): """Show the packages of a rpm group""" @@ -1585,10 +1598,10 @@ def cmd_group_owner(options): } def get_output(source, maints, packages): - for package in list(packages.keys()): + for package_name in list(packages.keys()): maint = maints.get(source, "?") - yield "\t".join((maint, source, ",".join(sorted(packages[package])))) + yield "\t".join((maint, source, ",".join(sorted(packages[package_name])))) first = True for group in list(packages.keys()): @@ -1632,7 +1645,7 @@ def cmd_cleanup(_): except IndexError: print(os.path.join(root, path, 'SOURCES', 'sha1.lst')) # shutil.rmtree(os.path.join(root, path)) -# Downstream.checkout(path) +# Downstream.package(path).checkout() continue vcs = pysvn.Client() @@ -1664,10 +1677,10 @@ def cmd_ls(options): else: streams = sorted(streams) - for package, module, package_version, spec_version, _ in streams: - sys.stdout.write(package) + for package_name, module, package_version, spec_version, _ in streams: + sys.stdout.write(package_name) if options.spec: - sys.stdout.write('/SPECS/%s.spec' % package) + sys.stdout.write('/SPECS/%s.spec' % package_name) if options.upstream: sys.stdout.write("\t%s" % module) if options.show_version: @@ -1683,11 +1696,11 @@ def cmd_check_version(_): streams = join_streams(show_version=True) - for package, _, package_version, spec_version, _ in streams: + for package_name, _, package_version, spec_version, _ in streams: if package_version == spec_version: continue - sys.stdout.write(package) + sys.stdout.write(package_name) sys.stdout.write("\t%s\t%s" % (spec_version, package_version)) sys.stdout.write("\n") @@ -1696,9 +1709,9 @@ def cmd_check_latest(options): streams = join_streams(show_version=True) - for package, module, package_version, spec_version, _ in streams: + for package_name, module, package_version, spec_version, _ in streams: upgrade = set() - sys.stdout.write(package) + sys.stdout.write(package_name) sys.stdout.write("\t%s\t%s" % (spec_version, package_version)) safe_max_version = get_safe_max_version(spec_version, module=module) @@ -1731,49 +1744,50 @@ def cmd_check_latest(options): print() if 'S' in upgrade and options.submit and not Downstream.DISTRO: - cmd = ['mgagnome', 'increase', package, safe_version] - subprocess.call(cmd, cwd=Downstream.package_path(package)) + cmd = ['mgagnome', 'increase', package_name, safe_version] + subprocess.call(cmd, cwd=Downstream.package(package_name).path) def cmd_patches(options): """List files with extension .patch or .diff as found in the source rpms""" root = os.path.expanduser(Downstream.PKGROOT) - for package, module, _, _, downstream_files in sorted(join_streams()): + for package_name, module, _, _, downstream_files in sorted(join_streams()): for filename in downstream_files: if '.patch' in filename or '.diff' in filename: - this_patch = Patch(os.path.join(root, package, "SOURCES", filename), show_path=options.path) - print("\t".join((module, package, str(this_patch)))) + this_patch = Patch(os.path.join(root, package_name, "SOURCES", filename), show_path=options.path) + print("\t".join((module, package_name, str(this_patch)))) def cmd_check_prep(options): """Check if the %prep stage can be ran sucessfully This runs the %prep stage without dependencies""" - spec = Downstream.package_spec(options.package) - spec.check_and_update_patches() + package = Downstream.package(options.package) + package.spec.check_and_update_patches() def _cmd_clean_spec_multi(args): - options, package = args + options, package_name = args - print(package) - cwd = Downstream.package_path(package) + print(package_name) + package = Downstream.package(package_name) + cwd = package.path - path = os.path.join(cwd, "SPECS", "%s.spec" % package) + path = os.path.join(cwd, "SPECS", "%s.spec" % package_name) if not os.path.exists(path): try: - Downstream.checkout(package) + package.checkout() except subprocess.CalledProcessError: - print('WARNING: Cannot check package %s. Skipping.' % package, file=sys.stderr) + print('WARNING: Cannot check package %s. Skipping.' % package_name, file=sys.stderr) return False - spec = Downstream.package_spec(package) + spec = package.spec try: if not spec.ensure_no_local_changes(options.force): return False except subprocess.CalledProcessError: # Package was probably not checked out or something - print("ERROR: cannot clean spec file for %s" % package, file=sys.stderr) + print("ERROR: cannot clean spec file for %s" % package_name, file=sys.stderr) return False made_changes = False @@ -1954,7 +1968,7 @@ def _cmd_clean_spec_multi(args): # If we made it this far, checkin the changes if made_changes: if options.doit: - Downstream.checkin(package, spec.changes, cwd=cwd) + package.checkin(spec.changes, cwd=cwd) else: # show the diff and undo all changes print(spec.changes) @@ -1963,19 +1977,20 @@ def _cmd_clean_spec_multi(args): return made_changes def _cmd_check_spec_multi(args): - _, package = args - cwd = Downstream.package_path(package) + _, package_name = args + package = Downstream.package(package_name) + cwd = package.path - path = os.path.join(cwd, "SPECS", "%s.spec" % package) + path = os.path.join(cwd, "SPECS", "%s.spec" % package_name) if not os.path.exists(path): return False - spec = Downstream.package_spec(package) + spec = package.spec try: spec.patches except SpecFileError: - print('ERROR: Broken spec file for package %s' % package, file=sys.stderr) + print('ERROR: Broken spec file for package %s' % package_name, file=sys.stderr) return False return True @@ -1990,12 +2005,12 @@ def cmd_check_spec(options): if options.debug: - for package in packages: - _cmd_check_spec_multi((options, package)) + for package_name in packages: + _cmd_check_spec_multi((options, package_name)) else: workers = os.cpu_count() or 4 with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: - executor.map(_cmd_check_spec_multi, ((options, package) for package in packages)) + executor.map(_cmd_check_spec_multi, ((options, package_name) for package_name in packages)) @@ -2013,15 +2028,15 @@ def cmd_clean_spec(options): if options.debug: - for package in packages: - _cmd_clean_spec_multi((options, package)) + for package_name in packages: + _cmd_clean_spec_multi((options, package_name)) else: workers = os.cpu_count() or 4 # Hack: warm alternative provides cache if options.convert_br: Downstream.alternative_provides('XXXX') with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as executor: - executor.map(_cmd_clean_spec_multi, ((options, package) for package in packages)) + executor.map(_cmd_clean_spec_multi, ((options, package_name) for package_name in packages)) def cmd_new_release(options): """Increase the release of a package (source rpm) and submit that to the buildsystem @@ -2033,19 +2048,20 @@ def cmd_new_release(options): # Determine the package name if options.upstream: try: - package = Downstream().get_downstream_from_upstream(pkg, options.version)[0] + package_name = Downstream().get_downstream_from_upstream(pkg, options.version)[0] except ValueError as exc: print("ERROR: %s" % exc, file=sys.stderr) success = False continue else: - package = pkg + package_name = pkg - cwd = Downstream.package_path(package) + package = Downstream.package(package_name) + cwd = package.path # Checkout package to ensure the checkout reflects the latest changes try: - Downstream.checkout(package) + package.checkout() except subprocess.CalledProcessError: subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd) success = False @@ -2053,12 +2069,12 @@ def cmd_new_release(options): # SpecFile class handles the actual version+release change # XXX - module should reflect upstream name, this gives it the package name - spec = Downstream.package_spec(package) + spec = package.spec cur_release = spec.release try: new_release = int(cur_release)+1 except ValueError: - print("ERROR: Cannot increase the release for package %s" % pkg, file=sys.stderr) + print("ERROR: Cannot increase the release for package %s" % package_name, file=sys.stderr) success = False continue cur_version = spec.version @@ -2076,11 +2092,11 @@ def cmd_new_release(options): try: # If we made it this far, checkin the changes - Downstream.checkin(package, spec.changes, cwd=cwd) + package.checkin(spec.changes, cwd=cwd) # Submit is optional if options.submit: - Downstream.submit(package) + package.submit() except subprocess.CalledProcessError: success = False continue @@ -2096,26 +2112,28 @@ def cmd_package_new_version(options): # Determine the package name if options.upstream: try: - package = Downstream().get_downstream_from_upstream(options.package, options.version)[0] + package_name = Downstream().get_downstream_from_upstream(options.package, options.version)[0] except ValueError as exc: print("ERROR: %s" % exc, file=sys.stderr) sys.exit(1) else: - package = options.package + package_name = options.package + + package = Downstream.package(package_name) # Directories packages are located in - cwd = Downstream.package_path(package) + cwd = package.path # Checkout package to ensure the checkout reflects the latest changes try: - Downstream.checkout(package) + package.checkout() except subprocess.CalledProcessError: subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd) sys.exit(1) # SpecFile class handles the actual version+release change # XXX - module should reflect upstream name, this gives it the package name - spec = Downstream.package_spec(package) + spec = package.spec print("%s => %s" % (spec.version, options.version)) # XXX - Duplicate check as should not revert changes if specfile has already been changed @@ -2138,7 +2156,7 @@ def cmd_package_new_version(options): # If there are multiple sources, try to see if there is a preferred name # --> needed for metacity hash check (multiple tarball sources) if len(sources) > 1: - preferred_name = '%s-%s.tar.xz' % (package, options.version) + preferred_name = '%s-%s.tar.xz' % (package_name, options.version) if preferred_name in sources: sources = [preferred_name] @@ -2152,11 +2170,11 @@ def cmd_package_new_version(options): try: # If we made it this far, checkin the changes - Downstream.checkin(package, spec.changes, cwd=cwd) + package.checkin(spec.changes, cwd=cwd) # Submit is optional if options.submit: - Downstream.submit(package) + package.submit() except subprocess.CalledProcessError: sys.exit(1) @@ -2251,13 +2269,13 @@ def cmd_parse_ftp_release_list(options): time.sleep(secs) error = False - for package in packages: + for package_name in packages: cmd = ['mgagnome', 'increase', '--hash', hexdigest] if options.submit: cmd.append('--submit') if options.force: cmd.append('--force') - cmd.extend((package, version)) + cmd.extend((package_name, version)) if subprocess.call(cmd, stdout=stdout, stderr=stderr): error = True -- cgit v1.2.1