From b2217184cb9d12220792398f0c485f60cc983b3b Mon Sep 17 00:00:00 2001 From: Olav Vitters Date: Mon, 26 Jan 2015 14:33:27 +0100 Subject: use concurrent.futures instead of multiprocessing --- mgagnome | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/mgagnome b/mgagnome index 4dbe61d..7fed651 100755 --- a/mgagnome +++ b/mgagnome @@ -55,7 +55,7 @@ import shlex # check-latest import requests -import multiprocessing +import concurrent.futures SLEEP_INITIAL=180 SLEEP_REPEAT=30 @@ -980,7 +980,7 @@ class Upstream(object): # Get the files usock = urlopen.open(self.URL) parser = urllister() - parser.feed(usock.read()) + parser.feed(usock.read().decode('utf-8')) usock.close() parser.close() files = parser.urls @@ -1164,16 +1164,15 @@ def cmd_co_multi(args): pass def cmd_co(options, parser): - p = multiprocessing.Pool(5) - - if options.all: - packages = ((package, package) for package in Downstream().packages) - else: - packages = ((l[0], "%s => %s" % (l[0], l[1])) for l in sorted(join_streams())) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + if options.all: + packages = ((package, package) for package in Downstream().packages) + else: + packages = ((l[0], "%s => %s" % (l[0], l[1])) for l in sorted(join_streams(auto_update=False))) - p.map(cmd_co_multi, packages) + executor.map(cmd_co_multi, packages) -def join_streams(show_version=False, only_diff_version=False): +def join_streams(show_version=False, only_diff_version=False, auto_update=True): root = os.path.expanduser(Downstream.PKGROOT) upstream = Upstream().names @@ -1184,9 +1183,9 @@ def join_streams(show_version=False, only_diff_version=False): for package in list(downstream.tarballs[module].keys()): package_version = downstream.tarballs[module][package] spec_version = None - if show_version or only_diff_version: - cwd = os.path.join(root, package) + cwd = os.path.join(root, package) + if show_version or only_diff_version: # ensure package is checked out if not os.path.exists(cwd): try: @@ -1199,15 +1198,13 @@ def join_streams(show_version=False, only_diff_version=False): except subprocess.CalledProcessError: spec_version = 'N/A' - # in case upstream version is newer, update checkout - if package_version != spec_version and version_cmp(package_version, spec_version) == 1: - try: - downstream.co(package) - spec_version = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package), module=module).version - except subprocess.CalledProcessError: - pass - - + # in case upstream version is newer, update checkout + if auto_update and package_version != spec_version and version_cmp(package_version, spec_version) == 1: + try: + downstream.co(package) + spec_version = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package), module=module).version + except subprocess.CalledProcessError: + pass if only_diff_version and package_version == spec_version: continue @@ -1467,8 +1464,8 @@ def cmd_clean_spec(options, parser): else: packages = options.package if len(options.package) else (l[0] for l in join_streams()) - p = multiprocessing.Pool(5) - p.map(cmd_clean_spec_multi, ((options, package) for package in packages)) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + executor.map(cmd_clean_spec_multi, ((options, package) for package in packages)) def cmd_package_new_version(options, parser): # Determine the package name -- cgit v1.2.1