path: root/mgagnome

#!/usr/bin/python3 -u
"""Mageia GNOME commands."""

# A lot of the code comes from ftpadmin, see
#   https://git.gnome.org/browse/sysadmin-bin/tree/ftpadmin
# Written by Olav Vitters

from functools import wraps, lru_cache, cached_property

# basic modules:
import os
from pathlib import Path
import sys
import re
import subprocess

# command line parsing, error handling:
import argparse
import errno

# overwriting files by moving them (safer):
import tempfile
import shutil

# getting links from HTML document:
from html.parser import HTMLParser
import urllib.request
import urllib.error
import urllib.parse

# for checking hashes
import hashlib

# for parsing ftp-release-list emails
import email
from email.mime.text import MIMEText

# to be able to sleep for a while
import time

# packages --sort
import itertools

# automatically dropping merged patches
import shlex

import concurrent.futures

# for debugging output
import pprint
import logging

# check-latest
import requests

# version comparison:
import rpm

SLEEP_INITIAL = 180
SLEEP_REPEAT = 30
SLEEP_TIMES = 30

_re_majmin = re.compile(r'^([0-9]+\.[0-9]+).*')
_re_version = re.compile(r'([-.]|\d+|[^-.\d]+)')



def retry(exceptions, tries=4, delay=3, backoff=2, logger=None):
    """Retry calling the decorated function using an exponential backoff.

    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry

    :param exceptions: the exception to check. may be a tuple of
        exceptions to check
    :type exceptions: Exception or tuple
    :param tries: number of times to try (not retry) before giving up
    :type tries: int
    :param delay: initial delay between retries in seconds
    :type delay: int
    :param backoff: backoff multiplier e.g. value of 2 will double the delay
        each retry
    :type backoff: int
    :param logger: logger to use. If None, print
    :type logger: logging.Logger instance
    """
    def deco_retry(func):

        @wraps(func)
        def func_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            while mtries > 1:
                try:
                    return func(*args, **kwargs)
                except exceptions as exc:
                    msg = "%s, Retrying in %d seconds..." % (str(exc), mdelay)
                    if logger:
                        logger.warning(msg)
                    else:
                        print(msg)
                    time.sleep(mdelay)
                    mtries -= 1
                    mdelay *= backoff
            return func(*args, **kwargs)

        return func_retry  # true decorator

    return deco_retry


def version_cmp(version_a, version_b):
    """Compares two versions

    Returns
      -1 if a < b
      0  if a == b
      1  if a > b
    """

    return rpm.labelCompare(('1', version_a, '1'), ('1', version_b, '1'))

def get_latest_version(versions, max_version=None):
    """Gets the latest version number

    if max_version is specified, gets the latest version number before
    max_version"""
    latest = None
    for version in versions:
        if (latest is None or version_cmp(version, latest) > 0) \
           and (max_version is None or version_cmp(version, max_version) < 0):
            latest = version
    return latest

MAJOR_VERSIONS = {
    # NAMES MUST BE IN LOWERCASE!
    'networkmanager': set(('0.9',)),
    'networkmanager-applet': set(('0.9',)),
    'networkmanager-openconnect': set(('0.9',)),
    'networkmanager-openvpn': set(('0.9',)),
    'networkmanager-pptp': set(('0.9',)),
    'networkmanager-vpnc': set(('0.9',))
}

def get_majmin(version, module=None):
    """Return a tuple with major and minor versions given a version"""
    nrs = version.split('.')

    if module and module.lower() in MAJOR_VERSIONS:
        module_versions = [version.split(".") for version in MAJOR_VERSIONS[module.lower()]]

        nrstest = nrs[:]

        while len(nrstest) >= 2:
            if nrstest in module_versions:
                return (".".join(nrs[:len(nrstest)]), nrs[len(nrstest)])

            nrstest.pop()

    return (nrs[0], nrs[1])


def get_safe_max_version(version, module=None):
    """Provide the maximum version a module could safely be increased to

    This assumed the module is uses symantic versioning"""
    match = _re_majmin.match(version)
    if version is None or not match:
        return None

    majmin = get_majmin(match.group(1), module)

    min_nr = int(majmin[1])

    # Add 2 for stable releases, 1 for unstable
    min_nr += 1 if min_nr % 2 == 0 else 2

    return "%s.%d" % (majmin[0], min_nr)

def judge_version_increase(version_old, version_new, module=None):
    """Judge quality of version increase:

    Returns a tuple containing judgement and message

    Judgement:
    Less than 0: Error
    0 to 4: Better not
    5+: Ok"""
    versions = (version_old, version_new)

    # First do a basic version comparison to ensure version_new is actually newer
    compare = version_cmp(version_new, version_old)

    if compare == 0:
        # 1.0.0 -> 1.0.1
        return (-2, "Already at version %s!" % (version_old))

    if compare != 1:
        # 1.0.1 -> 1.0.0
        return (-3, "Version %s is older than current version %s!" % (version_new, version_old))

    # Version is newer, but we don't want to see if it follows the GNOME versioning scheme
    majmins = [get_majmin(ver, module) for ver in versions if _re_majmin.match(ver) is not None]

    if len(majmins) == 1:
        return (-1, "Version number scheme changes: %s" % (", ".join(versions)))

    if len(majmins) == 0:
        return (0, "Unsupported version numbers: %s" % (", ".join(versions)))

    # Follows GNOME versioning scheme
    # Meaning: x.y.z
    #          x = major
    #          y = minor  : even if stable
    #          z = micro

    # Major+minor the same? Then go ahead and upgrade!
    if majmins[0] == majmins[1]:
        # Majmin of both versions are the same, looks good!
        # 1.1.x -> 1.1.x or 1.0.x -> 1.0.x
        return (10, None)

    # Check/ensure major version number is the same
    if majmins[0][0] != majmins[1][0]:
        # 1.0.x -> 2.0.x
        return (1, "Major version number increase")

    # Minor indicates stable/unstable
    devstate = (int(majmins[0][1]) % 2 == 0, int(majmins[1][1]) % 2 == 0)

    # Upgrading to unstable is weird
    if not devstate[1]:
        if devstate[0]:
            # 1.2.x -> 1.3.x
            return (1, "Stable to unstable increase")

        # 1.3.x -> 1.5.x
        return (3, "Unstable to unstable version increase")

    # Unstable => stable is always ok
    if not devstate[0]:
        # 1.1.x -> 1.2.x
        return (5, "Unstable to stable")

    # Can only be increase of minors from one stable to the next
    # 1.0.x -> 1.2.x
    return (4, "Stable version increase")

def line_input(file):
    """Iterate over a file and ignore any newline"""
    for line in file:
        if line[-1] == '\n':
            yield line[:-1]
        else:
            yield line

def distinct(iterable, keyfunc=None):
    """Iterate over an iterable and only return unique/distinct iterables"""
    seen = set()
    for item in iterable:
        key = item if keyfunc is None else keyfunc(item)
        if key not in seen:
            seen.add(key)
            yield item

class URLLister(HTMLParser):
    """Parse links from HTML"""
    # pylint: disable=abstract-method

    def reset(self):
        HTMLParser.reset(self)
        self.urls = []

    def handle_starttag(self, tag, attrs):
        if tag == 'a':
            href = [v for k, v in attrs if k == 'href']
            if href:
                self.urls.extend(href)

def _is_valid_hash(path, algo, hexdigest):
    if algo not in hashlib.algorithms_available:
        raise ValueError("Unknown hash algorithm: %s" % algo)

    local_hash = getattr(hashlib, algo)()

    with path.open('rb') as fp_file:
        data = fp_file.read(32768)
        while data:
            local_hash.update(data)
            data = fp_file.read(32768)

    return local_hash.hexdigest() == hexdigest

re_clean_1 = re.compile(r'\[[^]()]+\]$')
def clean_pkgconfig_prov(prov):
    prov = re_clean_1.sub('', prov)
    return prov


class SpecFileError(Exception):
    """Used for problems in the spec file"""


class SpecFile():
    """Functions related to a spec file"""
    re_update_version = re.compile(
        r'^(?P<pre>Version[ \t]*:\s*)(?P<version>.+)(?P<post>\s*)$',
        re.MULTILINE + re.IGNORECASE
    )
    re_update_release = re.compile(
        r'^(?P<pre>Release[ \t]*:\s*)(?P<release>%mkrel [0-9.]+)(?P<post>\s*)$',
        re.MULTILINE + re.IGNORECASE
    )
    re_update_patch = re.compile(
        r'^(?P<pre>Patch0*?)(?P<nr>[0-9]*)(?P<pre2>[ \t]*:\s*)(?P<patch>.+)(?P<post>\s*)\n',
        re.MULTILINE + re.IGNORECASE
    )
    re_br_part = re.compile(
        r'(?P<br>[^\s%{},<>=][^\s%{},<>=]*)\b(?P<post>\s*(?:(?P<operator>=|>=|<=|=<|=>|>|<)\s*' \
        r'(?P<version>[^\s%{},]+|\%\{[^\s{%}]+\}|\%[^\s%{},]+)\b)?)'
    )
    #re_update_br = re.compile(
    #    r'^(?P<pre>BuildRequires:\s*)(?P<br>[^\s%{},]+?)(?P<post>\s*(?:(?:[<>]=?|=)\s+[^\s%{},]+?)?\s*\n)',
    #    re.MULTILINE + re.IGNORECASE
    #)
    re_update_br = re.compile(
        r'^(?P<pre>BuildRequires[ \t]*:\s*)(?P<br>[^\s%{},]+?)' \
        r'(?P<post>\s*(?:(?:[<>]=?|=)\s+(?:[^\s%{},]+|\%\{[^\s{%}]+\}|\%[^\s%{},]+))?\s*\n)',
        re.MULTILINE + re.IGNORECASE
    )
    #re_update_br_unsplit = re.compile(
    #    r'^(?P<pre>BuildRequires:\s*)(?P<unsplit>[^\n,]+,[^\n]*)(?P<post>\s*\n)',
    #    re.MULTILINE + re.IGNORECASE
    #)

    re_update_br_fix_operator = re.compile(
        r'^(?P<pre>BuildRequires[ \t]*:\s*[^\n]*)(?P<operator>=<|=>)(?P<post>[^\n]+)\n',
        re.MULTILINE + re.IGNORECASE
    )
    re_update_br_unsplit = re.compile(
        r'^(?P<pre>BuildRequires[ \t]*:\s*)(?P<unsplit>(?:%s,?(?:[ \t\f\v]+|$)){2,})' \
        r'(?P<unsplitpost>\n)' % (re_br_part.pattern,),
        re.MULTILINE + re.IGNORECASE
    )

    def __init__(self, path, package=None, module=None):
        self.path = path
        self._package = package
        self.module = package.name if package else module

        if package:
            self.cwd = package.path
        else:
            self.cwd = path.parent
            # If spec file is located in SPECS directory, cwd should be 1
            # directory higher
            if self.cwd.name == 'SPECS':
                self.cwd = self.cwd.parent

        # WARNING: Requires Python 3.7+ as that version guarantees dict ordering
        self._changes = dict()
        self._should_rebuild = False
        self._changed_spec_file = False

    @property
    def changes(self):
        """Description of the changes made to the spec file"""
        return ''.join(self._changes.keys()) \
            if len(self._changes) == 1 \
            else "\n".join(("- %s" % change for change in self._changes))

    @property
    def made_changes(self):
        """Determine if the spec file was changed"""
        return self._changed_spec_file

    @property
    def version(self):
        """Provide the version as parsed by rpm"""
        cmd = ["rpm", "--define", "_topdir %s" % self.cwd, "--specfile", self.path,
               "--queryformat", "%{VERSION}\n"]
        return subprocess.check_output(cmd, encoding='utf-8').splitlines()[0]

    @property
    def should_rebuild(self):
        """Determine if the package should be rebuilt"""
        return self._should_rebuild

    @property
    def release(self):
        """Provide the release as parsed by rpm"""
        cmd = ["rpm", "--define", "%dist %nil", "--define", "_topdir %s" % self.cwd,
               "--specfile", self.path, "--queryformat", "%{RELEASE}\n"]
        return subprocess.check_output(cmd, encoding='utf-8').splitlines()[0]

    def _sources_and_patches(self, flag=None):
        os.chdir(self.cwd)
        rpm.delMacro("_topdir")
        rpm.addMacro("_topdir", str(self.cwd))
        trans_set = rpm.ts()
        spec = trans_set.parseSpec(str(self.path))
        try:
            srclist = spec.sources if isinstance(spec.sources, (list, tuple)) \
                            else spec.sources()
        except ValueError as exc:
            # Reraise this into a more specific exception
            raise SpecFileError from exc
        finally:
            # trans_set.parseSpec can affect changing of internal macros, e.g. redefining things like mkrel and so on
            # reload the config to fix this
            rpm.reloadConfig()
        return dict(
            (Path(name).name, [name, 0 if no == 2147483647 and flags == 2 else no])
            for name, no, flags in srclist if flag is None or flags == flag
        )

    @property
    def patches(self):
        """Return the patches"""
        return self._sources_and_patches(flag=2)

    @property
    def sources(self):
        """Return the sources"""
        return self._sources_and_patches(flag=1)

    def convert_buildrequires(self, explain_changes=False):
        """Converts BuildRequires into their preferred form

        Requires SpecFile to be initialized with its package argument"""

        made_changes = False
        re_prov_get_version = re.compile(r'^[^(]+\([^)]+-(?P<version>[0-9]+\.[0-9][0-9.]*)\)$')
        no_alt = set()
        no_change = {}

        buildreqs = self.buildrequires

        log = logging.getLogger(None)
        debug_enabled = log.isEnabledFor(log.DEBUG)

        convert_brs = {
            'pkgconfig': {
                'desc': 'convert -devel buildrequires into pkgconfig',
                'check_br': lambda req: req.endswith('-devel'),
                'check_provide': lambda prov: prov.startswith('pkgconfig('),
                'basereqs': lambda req: [req[:-len('-devel')]],
                'basereq_no_version': lambda basereqs: [basereq.rstrip('1234567890.') for basereq in basereqs
                                                        if basereq[-1] in '1234567890'],
                'versions_from_basereq': lambda basereqs: set((basereq[len(basereq.rstrip('01234567890.')):]
                                                               for basereq in basereqs if basereq[-1] in '1234567890')),
                'versions_basereq_extra': lambda versions: set(("%s.0" % version for version in versions
                                                                if '.' not in version)),
                'extra': lambda basereqs, versions: \
                    ['pkgconfig(%s)' % basereq for basereq in basereqs] +
                         ['pkgconfig(%s)' % basereq[len('lib'):]
                          if basereq.startswith('lib') else 'pkgconfig(lib%s)' % basereq
                          for basereq in basereqs] +
                         ['pkgconfig(%s-%s)' % (basereq, version) for basereq in basereqs for version in versions],
            },
            'perl': {
                'desc': 'convert perl- buildrequires into perl()',
                'check_br': lambda req: req.startswith('perl-'),
                'check_provide': lambda prov: prov.startswith('perl('),
                'basereqs': lambda req: [req[len('perl-'):]],
                'extra': lambda basereqs, versions: ['perl(%s)' % basereq.replace('-', '::') for basereq in basereqs],
            },
            # PySolFC.spec:BuildRequires:       python3-setuptools
            # $ rpm -q python3-setuptools --provides | grep python3dist
            # python3dist(setuptools)
            # python3dist(setuptools) = 46.1.3
            #
            # There's also provides such as:
            # python3.8dist(setuptools)
            # pythonegg(3)(setuptools)
            'python-pkg': {
                'disabled': True,
                'desc': 'convert python buildrequires into python3dist()',
                'check_br': lambda req: req.startswith('python3-'),
                'check_provide': lambda prov: prov.startswith('python3dist('),
                'basereqs': lambda req: [req[len('python3-'):]],
                'extra': lambda basereqs, versions: ['python3dist(%s)' % basereq for basereq in basereqs],
            },
            'python-egg': {
                'desc': 'convert pythonegg(3) into python3dist()',
                'check_br': lambda req: req.startswith('pythonegg(3)(') and req.endswith(')'),
                'check_provide': lambda prov: prov.startswith('python3dist('),
                'basereqs': lambda req: [req[len('pythonegg(3)('):-1]],
                'extra': lambda basereqs, versions: ['python3dist(%s)' % basereq for basereq in basereqs],
            },
        }

        for keys in convert_brs.values():
            if 'disabled' in keys and keys['disabled']:
                continue

            keys['changes'] = {}
            br_old = [r for r in list(buildreqs.keys()) if keys['check_br'](r)]
            if debug_enabled and br_old:
                pprint.pprint(br_old)
            for req in br_old:
                every_provides, every_ignored_provide = Downstream.alternative_provides(req)
                # XXX - document what clean_pkgconfig_prov is for
                #       maybe integrate clean_pkgconfig_prov in alternative_provides function?
                provides = [clean_pkgconfig_prov(prov)
                            for prov in every_provides
                            if keys['check_provide'](prov)]
                provides_ignored = [clean_pkgconfig_prov(prov)
                                    for prov in every_ignored_provide
                                    if keys['check_provide'](prov)]
                change_to = None
                if len(provides) == 1 and not provides_ignored:
                    if debug_enabled:
                        print("NOTICE: Only one available option, using %s" % provides[0])

                    change_to = provides[0]
                elif provides and 'extra' in keys:
                    # Determine base require (e.g. gtk+3.0-devel --> gtk+3.0)
                    basereqs = keys['basereqs'](req)

                    # Determine version matches
                    versions = set()
                    if 'versions_from_basereq' in keys:
                        # Determine if the basereq has a version at the end (e.g. gtk+3.0 --> 3.0)
                        versions.update(keys['versions_from_basereq'](basereqs))
                        if versions and 'basereq_no_version' in keys:
                            basereqs.extend(keys['basereq_no_version'](basereqs))
                            # Make it unique again, but keep the order
                            #
                            # This is done so that e.g. python3-devel changes to pkgconfig(python3),
                            # even if pkgconfig(python) might be available
                            basereqs = list(distinct(basereqs))
                        if 'versions_basereq_extra' in keys:
                            versions.update(keys['versions_basereq_extra'](versions))

                    if not versions:
                        # In case no versions were retrieved from the basereq,
                        # match with any version found from the alternative
                        # provides (heuristic matching)
                        #
                        # This is only done as a last resort to avoid matching
                        # e.g. gtk+3.0-devel --> pkgconfig(gtk+2.0)
                        for prov in provides:
                            for match in re_prov_get_version.finditer(prov):
                                if debug_enabled:
                                    print("NOTICE: Heuristically adding version %s from provide %s" \
                                            % (match.group('version'), prov))
                                versions.add(match.group('version'))

                    check_for = keys['extra'](basereqs, versions)

                    if debug_enabled and versions:
                        pprint.pprint(versions)

                    for check in check_for:
                        if check in provides:
                            if debug_enabled:
                                print("NOTICE: Matched: %s => %s" % (check, provides))
                            change_to = check
                            break

                if change_to is None and provides:
                    provides_no_versions = []
                    for prov in provides:
                        if re_prov_get_version.fullmatch(prov) is None:
                            provides_no_versions.append(prov)

                    if len(provides_no_versions) == 1 and not provides_ignored:
                        change_to = provides_no_versions[0]
                        if debug_enabled:
                            print("NOTICE: Only one available versionless option, using %s" % change_to)


                if provides:
                    if change_to is None:
                        no_change[req] = (provides, check_for)
                else:
                    no_alt.add(req)

                if change_to is not None:
                    keys['changes'][req] = change_to

        # Optionally explain the intended and failed buildrequire changes
        if explain_changes:
            if debug_enabled:
                for keys in convert_brs.values():
                    if 'changes' in keys and keys['changes']:
                        pprint.pprint(keys['changes'])

            if no_alt:
                print("WARNING: no alternatives found for: %s" % ", ".join(sorted(no_alt)))

            if no_change and debug_enabled:
                pprint.pprint(no_change)


        keys_with_changes = [keys for keys in convert_brs.values() if 'changes' in keys and keys['changes']]
# XXX - seems to cause false messages
#    if not keys_with_changes:
#        keys_with_changes.append({'changes': [], 'desc': 'unsplit BRs'})

        for keys in keys_with_changes:
            if self.update_br(keys['changes'], change_description=keys['desc']):
                made_changes = True

        return made_changes


    def clean_spec(self):
        """Clean the spec file of deprecated statements"""

        made_changes = False
        with self.path.open('r', encoding='utf-8') as fp_spec:
            data = fp_spec.read()

            made_changes, data = self._clean_spec_regexp(made_changes, data)
            made_changes, data = self._clean_spec_py_requires(made_changes, data)
            made_changes, data = self._clean_spec_defines(made_changes, data)
            made_changes, data = self._clean_spec_patches(made_changes, data)

            # Overwrite file with new version number
            if made_changes:
                self._changed_spec_file = True
                write_file(self.path, data)

        return made_changes


    def _clean_spec_regexp(self, made_changes, data):
        """Clean spec using regular expressions"""

        re_rm_buildroot = r'^(?:\s*\[[^\n\]\[]+\][ \t]+\&\&[ \t]+)?(?:rm|\%__rm|\%\{__rm\}) *' \
            r'(?:-rf|-fr|-r) *"?(?:[%$]buildroot|[%$]\{buildroot\}|[%$]\{buildroot\}|' \
            r'\$RPM_BUILDROOT|\$RPM_BUILD_ROOT|\$\{RPM_BUILD_ROOT\}|\$RPM_BUILD_DIR)"?/?[ \t]*\n'

        re_clean_spec = [
            # remove %defattr
            (
                'remove defattr',
                None,
                re.compile(r'(?P<keeppre>^\%files(?:[ \t]+[^\n]*)?\n(?:^\%doc [^\n]+\n)?)'
                           r'^\%defattr\s*\(- *, *root *, *root *(?:, *-)?\)\s*\n',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'remove cleaning buildroot in install',
                None,
                re.compile(r'(?P<keeppre>^\%install(?:[ \t]+[^\n]*)?\n)' + re_rm_buildroot + r'\n?',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'remove clean section',
                None,
                re.compile(r'^\%clean[ \t]*\n(?:' + re_rm_buildroot +
                           r')?\s*(?P<keeppost>(?:^#[^%\n]+\n)*^(?:\%files|\%post|\%pre|\%trigger|'
                           r'\%install|\%package|\%check|\%_font_pkg|$(?!.|\n)))',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'remove buildroot definition',
                None,
                re.compile(r'^BuildRoot[ \t]*:[^\n]+\n', re.MULTILINE + re.IGNORECASE)
            ),
            (
                'remove unneeded setup option',
                None,
                re.compile(r'^(?P<keeppre>\%setup -q)(?: -n|n) (?:\%name|\%\{name\})-'
                           r'(?:\%version|\%\{version\})(?P<keeppost>\n)',
                           re.MULTILINE + re.IGNORECASE)),
            (
                'https for download.gnome.org',
                r'\g<keeppre>https://\g<keeppost>',
                re.compile(r'^(?P<keeppre>Source[0-9]*[ \t]*:[^\n]+)http://(?P<keeppost>download.gnome.org/[^\n]+\n)',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'download.gnome.org instead of ftp.gnome.org',
                r'\g<keeppre>https://download.gnome.org\g<keeppost>',
                re.compile(r'^(?P<keeppre>Source[0-9]*[ \t]*:[^\n]+)'
                           r'(?:ftp|http|https)://ftp.gnome.org/pub/GNOME'
                           r'(?P<keeppost>/[^\n]+\n)',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'metacpan.org instead of www.cpan.org and search.cpan.org',
                r'\g<keeppre>https://metacpan.org/release/\g<keepmore>\g<keeppost>',
                re.compile(r'^(?P<keeppre>URL[ \t]*:[^\n]+)'
                           r'http://(?:search\.cpan\.org)/(?:dist|~[^/]+)/(?P<keepmore>[^/\n]+)/?[ \t]*'
                           r'(?P<keeppost>[^\n]*\n)',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'metacpan.org instead of www.cpan.org and search.cpan.org',
                r'\g<keeppre>https://cpan.metacpan.org/\g<keepmore>\g<keeppost>',
                re.compile(r'^(?P<keeppre>Source[0-9]*[ \t]*:[^\n]+)'
                           r'https?://(?:(?:www\.cpan\.org)(?:/CPAN)?|(?:search\.cpan\.org)/CPAN)/+'
                           r'(?P<keepmore>authors/id/|modules/by-module/|modules/by-authors/id/)'
                           r'(?P<keeppost>[^\n]+\n)',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'restrict what libraries are matched with major numbers',
                r'\g<keeppre>{,.*}',
                re.compile(r'^(?P<keeppre>%{_libdir}[^\n]+})\*$', re.MULTILINE)
            ),
            (
                'keep library matching using two lines',
                r'\g<keeppre>\n\g<keeppre>.*',
                re.compile(r'^(?P<keeppre>%{_libdir}[^\n]+})$\n(?P=keeppre)\{,\.\*\}$', re.MULTILINE)
            ),
            (
                'make use of autopatch',
                r'%autopatch -p1',
                re.compile(r'^%apply_patches$', re.MULTILINE)
            ),
            (
                'change configure2_5x macro to configure',
                r'%configure',
                re.compile(r'^%configure2_5x\b', re.MULTILINE)
            ),
            (
                'change make macro to make_build',
                r'\g<keeppre>%make_build',
                re.compile(r'^(?P<keeppre>[ \t]*)%make\b', re.MULTILINE)
            ),
            (
                'make_build already implies V=1 and VERBOSE=1, remove these',
                r'\g<keeppre>%make_build',
                re.compile(r'^(?P<keeppre>[ \t]*)%make_build(?:[ \t]+(?:V=1|VERBOSE=1))+\b', re.MULTILINE)
            ),
            # Can have side-effects, disable for now
            #(
            #    'change makeinstall_std macro to make_install',
            #    r'%make_install',
            #    re.compile(r'^%makeinstall_std\b', re.MULTILINE)
            #),
            (
                'change find_lang --with-help into --with-gnome',
                r'\g<keeppre> --with-gnome\g<keeppost>',
                re.compile(r'^(?P<keeppre>\s*\%find_lang[^\\\n]+) --with-help(?P<keeppost>[^\\\n]*\n)',
                           re.MULTILINE + re.IGNORECASE)
            ),
            (
                'change find_lang remove duplicate --with-gnome',
                None,
                re.compile(r'^(?P<keeppre>\%find_lang[^\\\n]+ --with-gnome) --with-gnome(?P<keeppost>[^\\\n]*\n)',
                           re.MULTILINE + re.IGNORECASE)
            ),
            # Use new Python macros
            ('use new Python macros', r'%py3_build', re.compile(r'^%{__python3} setup.py build$', re.MULTILINE)),

            ('use new Python macros', r'%py3_install', re.compile(r'^%{__python3} setup.py install$', re.MULTILINE)),

            ('use new Python macros', r'%{python3_version}', re.compile(r'%{py3ver}', re.MULTILINE)),
            (
                'make use of _mandir macro',
                r'%{_mandir}\g<keeppost>',
                # /usr/share/man/man1/test_whitespaces.1.xz
                re.compile(r'^(?:%{_datadir}/man|%_datadir/man|/usr/share/man)(?P<keeppost>/man[0-9]+/'
                           r'[a-zA-Z0-9]+[-a-zA-Z0-9_./]+\.[0-9]+[.a-z]*(?:\n|$))',
                           re.MULTILINE)
            ),
            (
                'make use of _infodir macro',
                r'%{_infodir}\g<keeppost>',
                # /usr/share/man/man1/test_whitespaces.1.xz
                re.compile(r'^(?:%{_datadir}/info|%_datadir/info|/usr/share/info)(?P<keeppost>/'
                           r'[a-zA-Z0-9]+[-a-zA-Z0-9_./]+\.info[.a-z]*(?:\n|$))',
                           re.MULTILINE)
            ),
            (
                # %{_mandir}/man1/*.xz
                # avoid adding a *
                'avoid assuming xz compression scheme',
                r'\g<keeppre>\g<keeppost>',
                re.compile(r'^(?P<keeppre>(?:%{_mandir}|%_mandir)/'
                           r'[a-zA-Z0-9]+[-@a-zA-Z0-9_./]+/\*)'
                           r'\.xz(?P<keeppost>\n|$)',
                           re.MULTILINE)
            ),
            (
                # %{_mandir}/man8/fxotune.8.xz
                # need to add a *
                'avoid assuming xz compression scheme',
                r'\g<keeppre>*\g<keeppost>',
                re.compile(r'^(?P<keeppre>(?:%doc )?(?:%{_mandir}|%_mandir)/'
                           r'[a-zA-Z0-9]+[-*@+:a-zA-Z0-9_./]+[-a-zA-Z0-9_./]'
                           r'(?:/(?:%{name}|%name|%{shortname})\.[0-9a-z]+)?'
                           r')'
                           r'\.xz(?P<keeppost>\n|$)',
                           re.MULTILINE)
            ),
        ]

        for reason, change_to, regexp in re_clean_spec:
            if change_to is None:
                change_to = ""
                if 'keeppre' in regexp.groupindex:
                    change_to += r'\g<keeppre>'
                if 'keeppost' in regexp.groupindex:
                    change_to += r'\g<keeppost>'
            data, subs = regexp.subn(change_to, data)
            if subs:
                made_changes = True
                self._changes['SILENT %s' % reason] = True

        return made_changes, data


    def _clean_spec_py_requires(self, made_changes, data):
        """Convert %py_requires and %py_requires -d"""

        re_convert_py_requires = [
            (
                'remove py_requires',
                ('python',),
                re.compile(r'^\%(?:py_requires|\{py_requires\})[ \t]*\n', re.MULTILINE)
            ),
            (
                'remove py_requires -d',
                ('python', 'python-devel'),
                re.compile(r'^\%(?:py_requires[ \t]+-d|\{py_requires[ \t]+-d\})[ \t]*\n', re.MULTILINE)
            ),
        ]

        # First figure out how a buildrequire is usually defined
        match = self.re_update_br.search(data)
        br_pre = match.group('pre') if match and match.group('pre') else "BuildRequires:\t"

        for reason, new_brs, regexp in re_convert_py_requires:
            match = regexp.search(data)
            if match:
                # Don't add multiple buildrequires
                change_to = ""
                brs_in_file = set()
                for match2 in self.re_update_br.finditer(data):
                    if match2.group('br') in new_brs:
                        brs_in_file.add(match2.group('br'))

                for buildreq in set(new_brs) - brs_in_file:
                    change_to += ''.join((br_pre, buildreq, "\n"))
                data, subs = regexp.subn(change_to, data)
                if subs:
                    made_changes = True
                    self._changes['SILENT %s' % reason] = True

        return made_changes, data


    def _clean_spec_defines(self, made_changes, data):
        # Convert:
        #   %define name SOMETHING
        #   name: %{name}
        # Into
        #   name: SOMETHING
        converted_defines = []
        for search_for in ('name', 'version', 'release', 'summary', 'Summary', 'group'):
            search_for_ignore_case = ''.join(("[%s%s]" % (letter, letter.swapcase()) for letter in search_for))
            re_spec = re.compile(
                r'^(?P<keeppre>' + re.escape(search_for_ignore_case) + r'[ \t]*:[ \t]*)(?:\%' +
                re.escape(search_for) + r'|\%\{' + re.escape(search_for) +
                r'\})(?P<keeppost>[ \t]*\n)', re.MULTILINE
            )
            re_variable = re.compile(
                r'^(?P<keeppre>\%define[ \t]+' + re.escape(search_for) +
                r'[ \t]+(?P<definition>[^\n]+?))(?P<keeppost>[ \t]*\n)',
                re.MULTILINE
            )

            match = re_variable.search(data)
            if match and match.group('definition') and len(re_variable.findall(data)) == 1:
                match2 = re_spec.search(data)
                if match2:
                    data, subs = re_spec.subn(
                        r'\g<keeppre>' + match.group('definition').replace('\\', '\\\\') + r'\g<keeppost>',
                        data
                    )
                    if subs:
                        made_changes = True
                        data, subs = re_variable.subn('', data)
                        converted_defines.append(search_for)

        if made_changes and converted_defines:
            data = data.lstrip()
            self._changes['SILENT remove variable definition(s) %s' % ", ".join(converted_defines)] = True

        return made_changes, data

    def _clean_spec_patches(self, made_changes, data):
        re_autopatch = re.compile(r'^[ \t]*\%autopatch(?:[ \t]+-p(?P<strip>[0-9]+))?$', re.MULTILINE)

        re_patch_header = re.compile(
            '^Patch(?P<nr>[0-9]*)[ \t]*:[ \t]*(?P<filename>[^\n]+)\n',
            re.MULTILINE + re.IGNORECASE
        )
        re_patch_any = re.compile(r'^[ \t]*\%patch(?P<nr>[0-9]*)', re.MULTILINE)
        re_patch_valid = re.compile(
            r'^[ \t+]*\%patch(?P<nr>[0-9]*)(?:[ \t]+-p(?P<strip>[0-9]+))?(?:[ \t]+-b[ \t]+\S+)?$\n?',
            re.MULTILINE
        )
        re_prep_patches = re.compile(
            r'^\%setup[^\n]+$(?:' + re_patch_valid.pattern +
            r'|^#[^%\n]+\n|^[ \t]*(?:%{_bindir}/|%_bindir)?autoreconf[ \t][^\n]+$|\s)+\n\%build',
            re.MULTILINE
        )

        give_patchnr = lambda match: (
            match.group('nr') if len(match.group('nr')) == 1 else match.group('nr').lstrip('0')
            ) if match.group('nr') else "0"

        # Make use of %apply_patches

        if re_autopatch.search(data):
            # Ignore if using %autopatch
            return made_changes, data

        patches = self.patches
        if not patches:
            return made_changes, data

        if len(patches) > 5:
            print("NOTICE: More than 5 patches, skipping package", file=sys.stderr)
            return made_changes, data

        if self.uses_autopatch:
            return made_changes, data

        # XXX -- apparently this is really inefficient with e.g. apache
        match2 = re_prep_patches.search(data)
        patch_nrs_header = set((give_patchnr(match) for match in re_patch_header.finditer(data)))
        patch_nrs_any = set((give_patchnr(match) for match in re_patch_any.finditer(data)))
        patch_nrs_valid = set(
            (give_patchnr(match) for match in re_patch_valid.finditer(match2.group(0)))
            ) if match2 else set()

        if not patch_nrs_header:
            # XXX -- weird, self.patches should've returned 0 already
            return made_changes, data

        if not patch_nrs_header == patch_nrs_any == patch_nrs_valid:
            print("NOTICE: Unable to automatically convert %s patches into %%autopatch "
                  "(header/patch/valid: %s, %s, %s)" % (self.module, len(patch_nrs_header), len(patch_nrs_any),
                                                        len(patch_nrs_valid)),
                  file=sys.stderr \
            )
            return made_changes, data

        patch_flags = set((
            0 if match.group('strip') is None else match.group('strip')
            for match in re_patch_valid.finditer(match2.group(0))
        ))

        if len(patch_flags) != 1:
            print("NOTICE: Unable to automatically convert spec file to use autopatch as"
                  "different -p / strip levels are used", file=sys.stderr)
            return made_changes, data

        # Whoot, we can convert!!
        change_to = "%%autopatch -p%s\n" % list(patch_flags)[0]
        prep, subs1 = re_patch_valid.subn(change_to.replace('\\', '\\\\'), match2.group(0), count=1)
        prep, subs2 = re_patch_valid.subn('', prep)
        if len(patch_nrs_valid) != subs1 + subs2:
            print("WARNING: Couldn't replace patches?!? Likely error in program logic", file=sys.stderr)
            return made_changes, data

        # First check if patches currently apply
        if not self.check_and_update_patches(check_only=True):
            return made_changes, data

        try:
            change_to = data.replace(match2.group(0), prep, 1)
            self._changed_spec_file = True
            write_file(self.path, change_to)

            # Validate patches still apply
            if self.check_and_update_patches(check_only=True):
                data = change_to
                self._changes['SILENT use autopatch'] = True
                made_changes = True
        finally:
            if not made_changes:
                write_file(self.path, data)

        return made_changes, data

    @property
    def buildrequires(self):
        """Get the BuildRequires of this spec file

        Parses the spec file to do this, so any macros are expanded"""

        rpm.delMacro("_topdir")
        rpm.addMacro("_topdir", str(self.cwd))
        trans_set = rpm.ts()
        try:
            spec = trans_set.parseSpec(str(self.path))
        except ValueError as exc:
            # Reraise this into a more specific exception
            raise SpecFileError from exc
        finally:
            # ts.parseSpec can affect changing of internal macros, e.g. redefining things like mkrel and so on
            # reload the config to fix this
            rpm.reloadConfig()

        requires = spec.sourceHeader[rpm.RPMTAG_REQUIRES]
        require_flags = spec.sourceHeader[rpm.RPMTAG_REQUIREFLAGS]
        require_ver = spec.sourceHeader[rpm.RPMTAG_REQUIREVERSION]

        buildreqs = {}
        for req, flag, ver in itertools.zip_longest(requires, require_flags, require_ver):
            # bitmask other than 15 means the require is (probably?) a require for a trigger or script
            if flag & 15 != flag:
                continue

            ver_cmp = ""
            if flag & rpm.RPMSENSE_LESS:
                ver_cmp += '<'
            if flag & rpm.RPMSENSE_GREATER:
                ver_cmp += '>'
            if flag & rpm.RPMSENSE_EQUAL:
                ver_cmp += '='

            buildreqs[req] = (ver_cmp, ver)
        return buildreqs

    @property
    def uses_autopatch(self):
        """Check if autopatch (or similar) is used"""
        return subprocess.call(['grep', '-Eq', '^%(apply_patches|autopatch|autosetup)', '--', self.path]) == 0

    def _revert_changes(self):
        """Revert uncommited changes made to spec file"""
        self._changes.clear()
        subprocess.check_call(["svn", "revert", "-R", self.cwd])

    def remove_patch(self, patchnr, info_patchname=None):
        """Remove a patch from the spec file"""
        # Uses string as well as numeric comparisons to determine which patch to remove
        nrs = [str(patchnr), patchnr]
        if patchnr == 0:
            nrs.append('')

        with self.path.open('r', encoding='utf-8') as fp_spec:
            data = fp_spec.read()

            data, subs = self.re_update_patch.subn(
                lambda match: '' if match.group('nr') in nrs or (match.group('nr').isdigit()
                                                                 and int(match.group('nr')) in nrs)
                else match.group(0), data
            )
            if not subs:
                print("ERROR: Could not remove patch nr %s!" % patchnr, file=sys.stderr)
                return False

            # Overwrite file with new version number
            self._changed_spec_file = True
            write_file(self.path, data)

            if info_patchname:
                self._changes['dropped merged patch %s' % info_patchname] = True
            else:
                self._changes['dropped merged patch %s' % patchnr] = True

        subprocess.check_call(['mgarepo', 'sync'], cwd=self.cwd)
        return True

    def check_and_update_patches(self, check_only=False, lines_in_log=15):
        """Check if patches still apply

        Remove any merged patches"""

        initial_patches = self.patches
        patches = initial_patches
        uses_autopatch = self.uses_autopatch if patches else False

        while True:
            try:
                # Check patches still apply
                subprocess.check_call(['bm', '-p', '--nodeps'], cwd=self.cwd)
            except subprocess.CalledProcessError:
                logfile = self.path.parent.joinpath(
                    'log.%s' % self.path.stem
                )

                failed_patch = None
                cmd = None
                cmd_output = []
                cmd_before = (None, None)
                # Determine the last command that failed
                if logfile.exists():
                    print(logfile)
                    with logfile.open('r', encoding='utf-8') as fp_logfile:
                        for line in line_input(fp_logfile):
                            if line.startswith('+ '):
                                cmd_before = (cmd, cmd_before)
                                cmd = line[2:]
                                cmd_output = []
                            else:
                                cmd_output.append(line)

                cmd_parsed = shlex.split(cmd) if cmd else []
                cmd_before_parsed = shlex.split(cmd_before[0]) if cmd_before[0] else []

                if not check_only and uses_autopatch and patches and cmd_parsed:
                    if Path(cmd_parsed[0]).name in ('patch', 'cat'):
                        if Path(cmd_parsed[-1]).exists():
                            failed_patch = Path(cmd_parsed[-1]).name
                        elif cmd_parsed[-1].startswith('-') and Path(cmd_before_parsed[-1]).exists():
# for %autopatch as well as %patch
#+ /usr/bin/cat /home/src/pkgs/gnome-getting-started-docs/SOURCES/gs-browse-web-firefox.page.patch
#+ /usr/bin/patch -p1 -s
                            failed_patch = Path(cmd_before_parsed[-1]).name

                    # Patch is merged if there is at least one 'ignored' line and no 'FAILED' line anywhere
                    has_ignored = False
                    has_reversed = False
                    has_failed = False
                    for line in cmd_output:
                        if 'FAILED' in line or 'File to patch:' in line:
                            has_failed = True
                            break

                        if 'ignored' in line:
                            has_ignored = True
                        elif 'saving rejects to file' in line:
                            has_failed = True
                            break

                        if 'Reversed (or previously applied) patch detected' in line:
                            has_reversed = True

                    if has_ignored and has_reversed and not has_failed:
                        # If patch was merged, drop it from spec file and rety
                        print("INFO: Patch has been merged: %s" % failed_patch, file=sys.stdout)
                        if failed_patch in patches:
                            if self.remove_patch(patches[failed_patch][1], failed_patch):
                                # try again
                                patches = self.patches
                                continue

                        print("ERROR: Problem removing merged patch: %s" % failed_patch, file=sys.stderr)
                        return False

                if cmd and len(cmd_output) > lines_in_log:
                    print('+ %s' % cmd, file=sys.stdout)
                    print("\n".join(cmd_output), file=sys.stdout)
                elif logfile.exists():
                    subprocess.call(['tail', '-n', str(lines_in_log), logfile])

                if failed_patch:
                    print("ERROR: Problem applying patch: %s" % failed_patch, file=sys.stderr)
                elif cmd:
                    print("ERROR: Problem in %%prep phase command: %s" % cmd, file=sys.stderr)
                elif patches:
                    print("ERROR: Problem applying patches and/or %prep phase", file=sys.stderr)
                else:
                    print("ERROR: Problem in %prep phase", file=sys.stderr)
                return False
            finally:
                # bm command doesn't cleanup after itself and the option it has to do that removes spec file + sources!
                buildpath = self.cwd.joinpath('BUILD', '%s-%s' % (self.module, self.version))
                if buildpath.exists():
                    shutil.rmtree(buildpath, ignore_errors=True)

            return True

    def update_br(self, changes, change_description='update buildrequirements'):
        """Update buildrequirement"""

        # XXX - doesn't handle buildrequires with version numbers :-(
        made_changes = False

        with self.path.open('r', encoding='utf-8') as fp_spec:
            data = fp_spec.read()

            # Change any "," in buildrequires into multiple lines
            data, subs = self.re_update_br_unsplit.subn(
                lambda match: ''.join((''.join((match.group('pre'), match2.group(0), match.group('unsplitpost')))
                                       for match2 in self.re_br_part.finditer(match.group('unsplit'))
                                       if match.group(0).strip() != '')),
                data
            )
            if subs:
                made_changes = True
                self._changes['SILENT one line per buildrequirement'] = True

            # Change =< and => operators into <= and >=
            data, subs = self.re_update_br_fix_operator.subn(
                lambda match: match.group(0).replace('=>', '>=').replace('=<', '<')
                if self.re_update_br.match(match.group(0).replace('=>', '>=').replace('=<', '<'))
                else match.group(0), data
            )
            if subs:
                made_changes = True
                self._changes['SILENT fix operator in buildrequires'] = True

            # Now update buildrequires if any
            data, subs = self.re_update_br.subn(
                lambda match: ''.join((match.group('pre'), changes[match.group('br')], match.group('post')))
                if match.group('br') in changes else match.group(0), data
            )

            if subs:
                made_changes = True
                self._changes['SILENT %s' % change_description] = True
            elif len(changes) != 0:
                print("ERROR: Could not update buildrequires for %s" % self.module, file=sys.stderr)
                print(changes)
                return False

            # Overwrite file with updated buildrequire
            self._changed_spec_file = True
            write_file(self.path, data)

        return made_changes


    MAX_JUDGEMENT = 5

    def ensure_no_local_changes(self, force=False):
        """Check if the repository has any local changes

        Can optionally force a clean checkout (by reverting any local
        changes). This should possibily be moved to Downstream class, or move
        this into a new Package class"""
        cmd = ["svn", "diff", self.cwd]
        svn_diff_output = subprocess.check_output(cmd, encoding='utf-8')
        if svn_diff_output != '':
            print(svn_diff_output)
            print("ERROR: Package has uncommitted changes!", file=sys.stderr)
            if not force:
                return False

            # Forcing package submission: revert changes
            try:
                print("WARNING: Force used; reverting svn changes", file=sys.stderr)
                self._revert_changes()
            except subprocess.CalledProcessError:
                return False

        return True

    def _check_can_update(self, force):
        data = None
        if not self.ensure_no_local_changes(force):
            return None

        with self.path.open('r', encoding='utf-8') as fp_spec:
            data = fp_spec.read()

            if data.count("%subrel") != 0:
                print("ERROR: %subrel found; don't know what to do!", file=sys.stderr)
                return None

            if data.count("%mkrel") != 1:
                print("ERROR: Multiple %mkrel found; don't know what to do!", file=sys.stderr)
                return None

        return data

    def update_release(self, release, reason, force=False):
        """Update release (usually for rebuilds)"""
        data = self._check_can_update(force)
        if data is None:
            return False

        # XXX - validate current release
        data, subs = self.re_update_release.subn(r'\g<pre>%%mkrel %s\g<post>' % release, data, 1)
        if subs != 1:
            print(data, file=sys.stdout)
            print("ERROR: Could not increase release!", file=sys.stderr)
            return False

        # Overwrite file with new release
        self._changed_spec_file = True
        write_file(self.path, data)

        self._changes['rebuild for %s' % reason] = True

        return True

    def update_version(self, version, force=False, max_judgement=MAX_JUDGEMENT):
        """Update version and reset release"""
        cur_version = self.version

        (judgement, msg) = judge_version_increase(cur_version, version, self.module)

        if judgement < 0:
            print("ERROR: %s!" % (msg), file=sys.stderr)
            return False

        if judgement < max_judgement:
            print("WARNING: %s!" % (msg))
            if not force:
                return False

        data = self._check_can_update(force)
        if data is None:
            return False

        data, subs = self.re_update_version.subn(r'\g<pre>%s\g<post>' % version, data, 1)
        if subs != 1:
            print("ERROR: Could not increase version!", file=sys.stderr)
            return False

        data, subs = self.re_update_release.subn(r'\g<pre>%mkrel 1\g<post>', data, 1)
        if subs != 1:
            print(data, file=sys.stdout)
            print("ERROR: Could not reset release!", file=sys.stderr)
            return False

        # Overwrite file with new version number
        self._changed_spec_file = True
        write_file(self.path, data)

        self._changes['new version %s' % version] = True

        # Verify that RPM also agrees that version number has changed
        if self.version != version:
            print("ERROR: Increased version to %s, but RPM doesn't agree!?!" % version, file=sys.stderr)
            return False

        # Try to download the new tarball various times and wait between attempts
        tries = 0
        while tries < SLEEP_TIMES:
            tries += 1
            if tries > 1:
                time.sleep(SLEEP_REPEAT * 2 ** (tries // 5))

            try:
                # Download new tarball
                subprocess.check_call(['mgarepo', 'sync', '-d'], cwd=self.cwd)
                # success, so exit loop
                break
            except subprocess.CalledProcessError as exc:
                # mgarepo sync returns 1 if the tarball cannot be downloaded
                if exc.returncode != 1:
                    self._revert_changes()
                    print("ERROR: Could not download tarball", file=sys.stderr)
                    return False
        else:
            # failed to download tarball
            self._revert_changes()
            print("ERROR: Could not download tarball", file=sys.stderr)
            return False

        return self.check_and_update_patches()

class Patch():
    """Do things with patches"""

    def __init__(self, path, show_path=False):
        """Path: path to patch (might not exist)"""
        self.path = path
        self.show_path = show_path

    def __str__(self):
        return self.path if self.show_path else self.path.name

    @cached_property
    def svn_author(self):
        """Return the svn author of this patch

        Makes use of svn log."""
        try:
            cmd = ['svn', 'log', '-q', "--", self.path]
            contents = subprocess.check_output(cmd, close_fds=True, encoding='utf-8').strip("\n").splitlines()

            for line in contents:
                if ' | ' not in line:
                    continue

                fields = line.split(' | ')
                if len(fields) >= 3:
                    return fields[1]
        except subprocess.CalledProcessError:
            pass

        return None


class Upstream():
    """Class handling anything related to upstream (meaning: GNOME)

    For now this is limited to:
      - finding the module names
      - finding latest version of a module"""


    URL = "https://download.gnome.org/sources/"
    limit = None

    def __init__(self):
        urlopen = urllib.request.build_opener()

        good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')

        # Get the files
        usock = urlopen.open(self.URL)
        parser = URLLister()
        parser.feed(usock.read().decode('utf-8'))
        usock.close()
        parser.close()
        files = parser.urls

        tarballs = set((filename.replace('/', '') for filename in files if good_dir.search(filename)))
        if self.limit is not None:
            tarballs.intersection_update(self.limit)

        self._names = tarballs

    @property
    def names(self):
        """Return the upstream names"""
        return self._names

    @classmethod
    @lru_cache(maxsize=4096)
    def versions(cls, module):
        """Return the possible versions for a given module"""
        versions = None

        url = '%s%s/cache.json' % (cls.URL, module)
        request = requests.get(url)
        json = request.json()
        if json is not None and len(json) > 2 and module in json[2]:
            versions = json[2][module]

        return versions

class Downstream():
    """Class handling anything related to downstream (meaning: Mageia)"""


    re_file = re.compile(
        r'^(?P<module>.*)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)' \
        r'\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$'
    )

    MEDIA = "Core Release Source"
    # PKGROOT will be overwritten (command line option)
    PKGROOT = Path('~/pkgs')
    DISTRO = None
    SECTION = None

    def __init__(self):
        cmd = ['urpmf', '--qf', '%name|%version|%files', '.', "--media", self.MEDIA]
        contents = subprocess.check_output(cmd, close_fds=True, encoding='utf-8').strip("\n").splitlines()

        srpm_files = {}
        module_srpm_tarballs = {}
        packages = set()

        for line in contents:
            try:
                srpm, version, filename = line.split("|")
            except SpecFileError:
                print(line, file=sys.stderr)
                continue

            packages.add(srpm)

            if '.tar' in filename:
                match = self.re_file.match(filename)
                if match:
                    fileinfo = match.groupdict()
                    module = fileinfo['module']

                    if module not in module_srpm_tarballs:
                        module_srpm_tarballs[module] = {}

                    if srpm in module_srpm_tarballs[module]:
                        # srpm seen before, check if version is newer
                        if version_cmp(module_srpm_tarballs[module][srpm], version) == 1:
                            module_srpm_tarballs[module][srpm] = version
                    else:
                        module_srpm_tarballs[module][srpm] = version

            if srpm not in srpm_files:
                srpm_files[srpm] = set()
            srpm_files[srpm].add(filename)

        self._packages = packages
        self.tarballs = module_srpm_tarballs
        self.files = srpm_files

    @property
    def packages(self):
        """Return all downstream package names"""
        return sorted(self._packages)

    @classmethod
    def package(cls, package):
        """Return an initialized package"""
        return Package(package, cls)

    _provide_to_alternate = {}
    @classmethod
    def alternative_provides(cls, search_for):
        """Give alternative provides for a certain provide

        Relies on urpmf. Results are cached. It will only provide alternatives if
        the alternative is only provided by one package. Meaning, if a pkgconfig(foo)
        is provided by 2 packages, the pkgconfig(foo) will NOT be given as an
        alternative.

        Sort of works like:
        $ urpmq --whatprovides $search_for --provides"""

        if not cls._provide_to_alternate:
            _provide_to_pkg = {}
            _pkg_to_provide = {}
            cmd = ['urpmf', "--qf", "%name\t%provides\t%arch", '.']
            for myline in subprocess.check_output(cmd, encoding='utf-8').splitlines():
                pkgname, pkgprovide, pkgarch = myline.split("\t")
                if pkgarch in ('src', 'i586'):
                    continue
                if '-debug' in pkgprovide:
                    continue

                if "[" in pkgprovide and pkgprovide.endswith("]"):
                    pkgprovidepart = pkgprovide.rstrip("]").partition("[")
                else:
                    pkgprovidepart = pkgprovide.partition("[")

                if pkgprovidepart[0] in _provide_to_pkg:
                    _provide_to_pkg[pkgprovidepart[0]].add(pkgname)
                else:
                    _provide_to_pkg[pkgprovidepart[0]] = set((pkgname,))

                if pkgname in _pkg_to_provide:
                    _pkg_to_provide[pkgname].add(pkgprovidepart[0])
                else:
                    _pkg_to_provide[pkgname] = set((pkgprovidepart[0],))

            provide_has_single = set()
            for key, stash in _provide_to_pkg.items():
                if len(stash) == 1:
                    provide_has_single.add(key)


            for key in provide_has_single:
                # Ignore some of the provides to optimize memory usage somewhat
                #
                # WARNING: This might need to be changed depending on how this
                # function is used
                if '(' in key and not ')(' in key:
                    continue

                for pkgname in _provide_to_pkg[key]:
                    for pkgprovide in _pkg_to_provide[pkgname]:
                        if not '(' in pkgprovide:
                            continue

                        if not key in cls._provide_to_alternate:
                            cls._provide_to_alternate[key] = (set(), set())

                        if pkgprovide in provide_has_single:
                            cls._provide_to_alternate[key][0].add(pkgprovide)
                        else:
                            cls._provide_to_alternate[key][1].add(pkgprovide)

        if search_for in cls._provide_to_alternate:
            return cls._provide_to_alternate[search_for]

        return (set(), set())

    def get_downstream_from_upstream(self, upstream, version):
        """Provide the downstream package(s) for a given upstream module name
        and version

        This will raise a ValueError exception in case a good match cannot be
        found"""


        if upstream not in self.tarballs:
            raise ValueError("No packages for upstream name: %s" % upstream)

        if len(self.tarballs[upstream]) == 1:
            return list(self.tarballs[upstream].keys())

        packages = {}
        for package_name in list(self.tarballs[upstream].keys()):
            package = self.package(package_name)
            # Checkout package to ensure the checkout reflects the latest changes
            try:
                package.checkout()
            except subprocess.CalledProcessError:
                raise ValueError("Multiple packages found and cannot checkout %s" % package_name)

            # Determine version from spec file
            try:
                packages[package_name] = package.spec.version
            except subprocess.CalledProcessError:
                raise ValueError("Multiple packages found and cannot determine version of %s" % package_name)

        # Return all packages reflecting the current version
        matches = [package_name for package_name in packages if packages[package_name] == version]
        if matches:
            return matches

        # Return all packages reflecting the version before the current version
        # - determine the newest version in any spec file
        latest_possible_version = get_latest_version(list(packages.values()))
        # - now get the latest version before the current version
        latest_version = get_latest_version(list(packages.values()), max_version=version)
        if latest_version is None:
            raise ValueError("Multiple packages found and all versions are newer than %s" % version)

        # - if the latest_possible_spec version is not the latest version, then
        #   ensure it follows get_safe_max_version
        if latest_version != latest_possible_version and version_cmp(get_safe_max_version(latest_version, upstream), \
                version) != 1:
            raise ValueError("Multiple packages found and unsafe version increase: %s (%s => %s)" % (
                upstream, latest_version, version) \
            )

        # - now really get the right packages
        matches = [package_name for package_name in packages if packages[package_name] == latest_version]
        if matches:
            return matches

        # Give up
        raise ValueError("Multiple packages found and cannot determine package for version %s" % version)

class Package:
    """Represents a downstream package"""

    def __init__(self, packagename, downstreamclass):
        self.name = packagename
        self._downstream = downstreamclass

    @property
    def path(self):
        """Provide the local checkout path for a given package

        Package might not be checked out yet!"""
        return self._downstream.PKGROOT.expanduser().joinpath(self.name)

    @cached_property
    def spec(self):
        """Return the SpecFile for a given package"""

        return SpecFile(self.spec_path, package=self, module=self.name)

    @cached_property
    def spec_path(self):
        """Return the expected location of the SpecFile"""

        return self.path.joinpath("SPECS", "%s.spec" % self.name)

    def ensure_checkout(self):
        """Ensure that the package is checked out"""
        if not self.spec_path.exists():
            try:
                self.checkout()
            except subprocess.CalledProcessError:
                print('WARNING: Cannot checkout package %s. Skipping.' % self.name, file=sys.stderr)
                return False

        return True

    @retry(subprocess.CalledProcessError)
    def checkout(self, cwd=None, spec_only=False):
        """Check out a package from the repository"""
        downstream = self._downstream
        if cwd is None:
            cwd = downstream.PKGROOT.expanduser()

        cmd = ['mgarepo', 'co']
        if downstream.DISTRO:
            cmd.extend(('-d', downstream.DISTRO))
        if spec_only:
            cmd.append('-s')
        cmd.append(self.name)
        return subprocess.check_call(cmd, stdin=subprocess.DEVNULL, cwd=cwd)

    @retry(subprocess.CalledProcessError)
    def checkin(self, changes, cwd=None):
        """Check in changes to the repository"""

        if cwd is None:
            cwd = self.path

        cmd = ['mgarepo', 'ci', '-m', changes]
        return subprocess.check_call(cmd, cwd=cwd)

    def submit(self, cwd=None):
        """Submit a package to the buildsystem

        If a specific distro release is chosen, the section will be set to
        core/updates_testing."""
        downstream = self._downstream
        if cwd is None:
            cwd = self.path

        cmd = ['mgarepo', 'submit']
        if downstream.DISTRO:
            cmd.extend(('--define', 'section=core/updates_testing', '-t', downstream.DISTRO))
        elif downstream.SECTION:
            cmd.extend(('--define', 'section={0}'.format(downstream.SECTION)))

        # retry submission various times, could be that some dependencies are being built at the same time
        @retry(subprocess.CalledProcessError, tries=10, delay=300, backoff=1.5)
        def _submit():
            subprocess.check_call(cmd, cwd=cwd)
        _submit()


def write_file(path, data):
    """Write to a file by creating a temporary file and renaming that to the
    new file"""

    with tempfile.NamedTemporaryFile(mode='w+t', dir=path.parent, delete=False, encoding='utf-8') as fdst:
        fdst.write(data)
        fdst.flush()
        os.replace(fdst.name, path)

def _cmd_checkout_multi(args):
    package_name, what_to_print, options = args

    print(what_to_print)

    try:
        Downstream.package(package_name).checkout(spec_only=options.spec_only)
    except subprocess.CalledProcessError:
        pass

def cmd_checkout(options):
    """Check out various packages in parallel"""
    if options.all:
        packages = ((package_name, package_name, options) for package_name in Downstream().packages)
    elif options.package:
        packages = ((package_name, package_name, options) for package_name in options.package)
    else:
        packages = ((l[0], "%s => %s" % (l[0], l[1]), options) for l in sorted(join_streams(auto_update=False)))

    if options.debug:
        for package_name in packages:
            _cmd_checkout_multi(package_name)
    else:
        with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
            executor.map(_cmd_checkout_multi, packages)

def join_streams(show_version=False, only_diff_version=False, auto_update=True):
    """Links upstream modules with downstream packages

    To figure out the downstream name of an upstream package, the Downstream
    class has logic to extract the name from a tarball file.

    If information is requested whereby the spec file is needed this function
    will automatically checkout any package"""

    upstream = Upstream().names
    downstream = Downstream()

    matches = upstream & set(downstream.tarballs.keys())
    for module in matches:
        for package_name in list(downstream.tarballs[module].keys()):
            package_version = downstream.tarballs[module][package_name]
            spec_version = None
            package = Downstream.package(package_name)
            cwd = package.path

            if show_version or only_diff_version:
                # ensure package is checked out
                if not cwd.exists():
                    try:
                        package.checkout()
                    except subprocess.CalledProcessError:
                        # XXX - ignoring packages which cannot be checked out
                        continue

                try:
                    spec_version = package.spec.version
                except subprocess.CalledProcessError:
                    spec_version = 'N/A'

                # in case upstream version is newer, update checkout
                if auto_update and package_version != spec_version and version_cmp(package_version, spec_version) == 1:
                    try:
                        package.checkout()
                        spec_version = package.spec.version
                    except subprocess.CalledProcessError:
                        pass

            if only_diff_version and package_version == spec_version:
                continue

            yield (package_name, module, package_version, spec_version, downstream.files[package_name])

def cmd_group_owner(options):
    """Show the packages of a rpm group"""


    groups = set(options.group)

    cmd = ["urpmf", "-F|", "--qf", "%group\t%name\t%sourcerpm\t%version\t%release", "."]
    output = [pkg.split("\t") for pkg in subprocess.check_output(cmd, encoding='utf-8').splitlines()]
    if not output:
        return

    # Filter by groups
    output = [pkg for pkg in output if pkg[0] in groups]
    if not output:
        return

    packages = {}
    for group, name, sourcerpm, version, release in output:
        if group not in packages:
            packages[group] = {}

        source = sourcerpm if sourcerpm else name
        for end in (".src.rpm", "-%s-%s" % (version, release)):
            if source.endswith(end):
                source = source[:len(source) - len(end)]

        if source not in packages[group]:
            packages[group][source] = set()

        packages[group][source].add(name)


    maints = {
        line.rpartition(" ")[::2]
        for line in subprocess.check_output(["mgarepo", "maintdb", "get"], encoding='utf-8').splitlines()
    }

    def get_output(source, maints, packages):
        for package_name in list(packages.keys()):
            maint = maints.get(source, "?")

            yield "\t".join((maint, source, ",".join(sorted(packages[package_name]))))

    first = True
    for group in list(packages.keys()):
        if first:
            first = False
        else:
            print("")
            print("")
        print(group)
        print("")

        for line in sorted(get_output(source, maints, packages[group])):
            print(line)

def cmd_cleanup(_):
    """Clean up the package root

    Removes things such as:
      - unneeded files in SOURCES (e.g. old tarballs)

    In future will also:
      - clean old directories in BUILDDIR
      - clean old files in RPMS and SRPMS"""


    root = Downstream.PKGROOT.expanduser()

#    packages = set(Downstream().packages)

    dirs = set((path for path in root.glob('*') if path.is_dir()))

#    dirs = dirs - packages

    import pysvn # pylint: disable=import-outside-toplevel
    dirs = [path for path in dirs if path.joinpath('SOURCES', 'sha1.lst').exists()
            and path.joinpath('SPECS', '%s.spec' % path.name).exists()]

    curtime = time.time()
    curtime_recent = curtime - (36 * 60 * 60)
    curtime_ancient = curtime - (180 * 24 * 60 * 60)

    for path in dirs:
        # Only select the packages which have not been modified for 36 hours
        mtime = path.joinpath('SPECS', '%s.spec' % path.name).stat().st_mtime
        if mtime > curtime_recent:
            print('INFO: Package was too recently modified: %s' % path.name)
            continue

        # Remove any packages not touched for 180 days
        if mtime < curtime_ancient:
            print('INFO: Package is ancient, removing: %s' % path.name)
            shutil.rmtree(path)
            continue


        # Cleanup BUILD and BUILDROOT directories
        for cleanup_dir in ('BUILD', 'BUILDROOT'):
            for cleanup_path in ((cleanup_path for cleanup_path in path.joinpath(cleanup_dir).glob('*') \
                                                   if cleanup_path.is_dir() \
                                                      and cleanup_path.stat().st_mtime < curtime_recent)):
                shutil.rmtree(cleanup_path)

        try:
            binaries = set((l.split('  ', 1)[1]
                            for l in path.joinpath('SOURCES', 'sha1.lst').open().read().splitlines()))
        except IndexError:
            print('ERROR: Problem parsing the sha1.lst of package %s' % path.name, file=sys.stderr)
#            shutil.rmtree(path)
#            Downstream.package(path).checkout()
            continue

        vcs = pysvn.Client()
        stats = [stat for stat in vcs.status(str(path.joinpath('SOURCES')), depth=pysvn.depth.immediates) # pylint: disable=no-member
                 if stat.text_status == pysvn.wc_status_kind.unversioned # pylint: disable=no-member
                 and Path(stat.path).name not in binaries]  # pylint: disable=no-member

        if stats:
            print(path)
            print(", ".join(Path(stat.path).name for stat in stats))
            print(stats)
            for stat in stats:
                stat_path = Path(stat.path)
                if stat_path.is_file():
                    stat_path.unlink()
                elif stat_path.is_dir():
                    shutil.rmtree(stat_path)

def cmd_ls(options):
    """Show upstream module names, downstream package names

    Optionally can even show version numbers"""

    streams = join_streams(show_version=options.show_version, only_diff_version=options.diff)
    if options.sort:
        # Sort packages on the line number in the file
        sort_helper = dict(list(zip(options.sort.read().splitlines(), itertools.count())))

        streams = sorted(streams, key=lambda a: (sort_helper.get(a[1], 9999), a[0]))
    else:
        streams = sorted(streams)

    for package_name, module, package_version, spec_version, _ in streams:
        sys.stdout.write(package_name)
        if options.spec:
            sys.stdout.write('/SPECS/%s.spec' % package_name)
        if options.upstream:
            sys.stdout.write("\t%s" % module)
        if options.show_version:
            sys.stdout.write("\t%s\t%s" % (spec_version, package_version))
        print()

def cmd_check_version(_):
    """Check for version mismatches between what's available in the spec
    version and as a source rpm.

    Example usage: locally check for failed builds"""


    streams = join_streams(show_version=True)

    for package_name, _, package_version, spec_version, _ in streams:
        if package_version == spec_version:
            continue

        sys.stdout.write(package_name)
        sys.stdout.write("\t%s\t%s" % (spec_version, package_version))
        sys.stdout.write("\n")

def cmd_check_latest(options):
    """Check if a package has a newer version upstream"""

    streams = join_streams(show_version=True)

    for package_name, module, package_version, spec_version, _ in streams:
        upgrade = set()
        sys.stdout.write(package_name)
        sys.stdout.write("\t%s\t%s" % (spec_version, package_version))

        safe_max_version = get_safe_max_version(spec_version, module=module)

        versions = Upstream.versions(module)
        if package_version != spec_version and spec_version != 'N/A' and package_version != 'N/A':
            upgrade.add('~')
        if versions:
            latest_version = get_latest_version(versions)
            safe_version = get_latest_version(versions, safe_max_version)

            cmp_latest = version_cmp(latest_version, spec_version)
            if cmp_latest < 0:
                latest_version = 'N/A'
                upgrade.add('l')
            elif cmp_latest > 0:
                upgrade.add('L')

            cmp_safe = version_cmp(safe_version, spec_version)
            if cmp_safe < 0:
                safe_version = 'N/A'
                upgrade.add('s')
            elif cmp_safe > 0:
                upgrade.add('S')

            sys.stdout.write("\t%s" % latest_version)
            sys.stdout.write("\t%s" % safe_version)
            sys.stdout.write("\t%s" % "".join(sorted(upgrade)))

        print()

        if 'S' in upgrade and options.submit and not Downstream.DISTRO:
            cmd = ['mgagnome', 'increase', package_name, safe_version]
            subprocess.call(cmd, cwd=Downstream.package(package_name).path)

def cmd_patches(options):
    """List files with extension .patch or .diff as found in the source rpms"""
    root = Downstream.PKGROOT.expanduser()

    for package_name, module, _, _, downstream_files in sorted(join_streams()):
        for filename in downstream_files:
            if '.patch' in filename or '.diff' in filename:

                this_patch = Patch(root.joinpath(package_name, 'SOURCES', filename), show_path=options.path)
                print("\t".join((module, package_name, str(this_patch))))

def cmd_check_prep(options):
    """Check if the %prep stage can be ran sucessfully

    This runs the %prep stage without dependencies"""
    package = Downstream.package(options.package)
    package.spec.check_and_update_patches()

def _cmd_clean_spec_multi(args):
    options, package_name = args

    print(package_name)
    package = Downstream.package(package_name)
    cwd = package.path

    if not package.ensure_checkout():
        return False

    spec = package.spec

    try:
        if not spec.ensure_no_local_changes(options.force):
            return False
    except subprocess.CalledProcessError:
        # Package was probably not checked out or something
        print("ERROR: cannot clean spec file for %s" % package_name, file=sys.stderr)
        return False

    made_changes = False

    # Convert perl- and -devel buildrequires into perl() and pkgconfig() requires
    if options.convert_br:
        if spec.convert_buildrequires(explain_changes=not options.doit):
            made_changes = True

    # Clean the spec file
    if spec.clean_spec():
        made_changes = True

    # If we made it this far, checkin the changes
    if made_changes:
        if options.doit:
            package.checkin(spec.changes, cwd=cwd)
        else:
            # show the diff and undo all changes
            print(spec.changes)
            spec.ensure_no_local_changes(force=True)

    return made_changes

def _cmd_check_spec_multi(args):
    _, package_name = args
    package = Downstream.package(package_name)

    if not package.spec_path.exists():
        return False

    spec = package.spec
    try:
        spec.patches
    except SpecFileError:
        print('ERROR: Broken spec file for package %s' % package_name, file=sys.stderr)
        return False

    return True


def cmd_check_spec(options):
    """Check if the spec files can be parsed by rpm"""
    if options.all:
        packages = Downstream().packages
    else:
        packages = options.package if options.package else (l[0] for l in join_streams())


    if options.debug:
        for package_name in packages:
            _cmd_check_spec_multi((options, package_name))
    else:
        workers = os.cpu_count() or 4
        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
            executor.map(_cmd_check_spec_multi, ((options, package_name) for package_name in packages))



def cmd_clean_spec(options):
    """Clean spec files

    Updates spec files to remove known deprecated statements from spec files

    Can optionally also change the BuildRequires"""

    if options.all:
        packages = Downstream().packages
    else:
        packages = options.package if options.package else (l[0] for l in join_streams())


    if options.debug:
        for package_name in packages:
            _cmd_clean_spec_multi((options, package_name))
    else:
        workers = os.cpu_count() or 4
        # Hack: warm alternative provides cache
        if options.convert_br:
            Downstream.alternative_provides('XXXX')
        with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as executor:
            executor.map(_cmd_clean_spec_multi, ((options, package_name) for package_name in packages))

def cmd_new_release(options):
    """Increase the release of a package (source rpm) and submit that to the buildsystem

    Also called a rebuild"""

    success = True
    for pkg in options.package:
        # Determine the package name
        if options.upstream:
            try:
                package_name = Downstream().get_downstream_from_upstream(pkg, options.version)[0]
            except ValueError as exc:
                print("ERROR: %s" % exc, file=sys.stderr)
                success = False
                continue
        else:
            package_name = pkg

        package = Downstream.package(package_name)
        cwd = package.path

        # Checkout package to ensure the checkout reflects the latest changes
        try:
            package.checkout()
        except subprocess.CalledProcessError:
            subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)
            success = False
            continue

        # SpecFile class handles the actual version+release change
        # XXX - module should reflect upstream name, this gives it the package name
        spec = package.spec
        cur_release = spec.release
        try:
            new_release = int(cur_release)+1
        except ValueError:
            print("ERROR: Cannot increase the release for package %s" % package_name, file=sys.stderr)
            success = False
            continue
        cur_version = spec.version
        print("%s-%s => %s-%s" % (cur_version, cur_release, cur_version, new_release))

        # XXX - Duplicate check as should not revert changes if specfile has already been changed
        if not spec.ensure_no_local_changes(options.force):
            success = False
            continue

        if not spec.update_release(new_release, options.reason, force=options.force):
            subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd, stdout=subprocess.DEVNULL)
            success = False
            continue

        try:
            # If we made it this far, checkin the changes
            package.checkin(spec.changes, cwd=cwd)

            # Submit is optional
            if options.submit:
                package.submit()
        except subprocess.CalledProcessError:
            success = False
            continue

    if not success:
        sys.exit(1)

def cmd_package_new_version(options):
    """Increase the version number of a package (source rpm)

    Can optionally submit the package to the buildsystem."""

    # Determine the package name
    if options.upstream:
        try:
            package_name = Downstream().get_downstream_from_upstream(options.package, options.version)[0]
        except ValueError as exc:
            print("ERROR: %s" % exc, file=sys.stderr)
            sys.exit(1)
    else:
        package_name = options.package

    package = Downstream.package(package_name)

    # Directories packages are located in
    cwd = package.path

    # Checkout package to ensure the checkout reflects the latest changes
    try:
        package.checkout()
    except subprocess.CalledProcessError:
        subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)
        sys.exit(1)

    # SpecFile class handles the actual version+release change
    # XXX - module should reflect upstream name, this gives it the package name
    spec = package.spec
    print("%s => %s" % (spec.version, options.version))

    # XXX - Duplicate check as should not revert changes if specfile has already been changed
    if not spec.ensure_no_local_changes(options.force):
        sys.exit(1)

    if not spec.update_version(options.version, force=options.force):
        # XXX - hack to automatically revert changes when auto upgrading from ftp release list
        if options.hexdigest is not None:
            subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd, stdout=subprocess.DEVNULL)
        sys.exit(1)

    # Check hash, if given
    if options.hexdigest is not None:
        sources = [name for name, value in spec.sources.items() if '://' in value[0]]
        if not sources:
            print("ERROR: Cannot determine source file (for hash check)!", file=sys.stderr)
            sys.exit(1)

        # If there are multiple sources, try to see if there is a preferred name
        # --> needed for metacity hash check (multiple tarball sources)
        if len(sources) > 1:
            preferred_name = '%s-%s.tar.xz' % (package_name, options.version)
            if preferred_name in sources:
                sources = [preferred_name]

        for filename in sources:
            path = cwd.joinpath("SOURCES", filename)
            if not _is_valid_hash(path, options.algo, options.hexdigest):
                print("ERROR: Hash file failed check for %s!" % path, file=sys.stderr)
                print("ERROR: Reverting changes!", file=sys.stderr)
                subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)
                sys.exit(1)

    try:
        # If we made it this far, checkin the changes
        package.checkin(spec.changes, cwd=cwd)

        # Submit is optional
        if options.submit:
            package.submit()

    except subprocess.CalledProcessError:
        sys.exit(1)

def cmd_parse_ftp_release_list(options):
    """Submit packages by parsing the GNOME ftp-release-list email

    """

    def _send_reply_mail(contents, orig_msg, to_addr, packages=None, error=False):
        """Send an reply email"""
        contents.seek(0)
        msg = MIMEText(contents.read(), _charset='utf-8')

        if error:
            # XXX - ugly
            contents.seek(0)
            lastline = contents.read().rstrip().splitlines()[-1]
            # Remove things like "ERROR: " and so on from the last line
            lastline = re.sub(r'^(?:[^ :]+:\s+)+', '', lastline)
            # Remove things like "   - " (youri output from mgarepo submit)
            lastline = re.sub(r'^\s+-\s+', '', lastline)
            subjecterror = " (ERROR: %s)" % lastline if lastline else " (ERROR)"
        else:
            subjecterror = ""

        if packages:
            subject = "%s %s%s" % (", ".join(packages), orig_msg['X-Module-Version'], subjecterror)
        else:
            subject = "Re: %s%s" % (orig_msg['Subject'], subjecterror)

        msg['Subject'] = subject
        msg['To'] = to_addr
        msg["In-Reply-To"] = orig_msg["Message-ID"]
        msg["References"] = orig_msg["Message-ID"]

        # Call sendmail program directly so it doesn't matter if the service is running
        cmd = ['/usr/sbin/sendmail', '-oi', '--']
        cmd.extend([to_addr])
        proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
        proc.stdin.write(msg.as_bytes())
        proc.stdin.flush()
        proc.stdin.close()
        proc.wait()


    msg = email.message_from_file(sys.stdin)

    if options.mail:
        stdout = tempfile.TemporaryFile(mode='w+t', encoding='utf-8')
        stderr = stdout
    else:
        stdout = sys.stdout
        stderr = sys.stderr

    try:
        module = msg['X-Module-Name']
        version = msg['X-Module-Version']
        hexdigest = msg['X-Module-SHA256-tar.xz']
    except KeyError as exc:
        print("ERROR: %s" % exc, file=stderr)
        if options.mail:
            _send_reply_mail(stdout, msg, options.mail, error=True)

        sys.exit(1)

    try:
        packages = Downstream().get_downstream_from_upstream(module, version)
    except (ValueError, SpecFileError) as exc:
        #print(bytes("ERROR: %s" % str(exc), 'UTF-8'), file=stderr)
        stderr.write("ERROR: %s" % str(exc))
        if options.mail:
            _send_reply_mail(stdout, msg, options.mail, error=True)

        sys.exit(1)

    if options.wait or options.fork:
        # maildrop aborts and will try to deliver after 5min
        # fork to avoid this
        if os.fork() != 0:
            sys.exit(0)

    if options.wait:
        # wait SLEEP_INITIAL after the message was sent
        secs = SLEEP_INITIAL
        parsed_date = email.utils.parsedate_tz(msg['Date'])
        if parsed_date is not None:
            msg_time = email.utils.mktime_tz(parsed_date)
            secs = SLEEP_INITIAL - (time.time() - msg_time)

        if secs > 0:
            time.sleep(secs)

    error = False
    for package_name in packages:
        cmd = ['mgagnome', 'increase', '--hash', hexdigest]
        if options.submit:
            cmd.append('--submit')
        if options.force:
            cmd.append('--force')
        cmd.extend((package_name, version))
        if subprocess.call(cmd, stdout=stdout, stderr=stderr):
            error = True

    if options.mail:
        _send_reply_mail(stdout, msg, options.mail, packages=packages, error=error)

def main():
    """Parse arguments and call main subfunction

    The Subfunctions are the functions whose name starts with cmd_"""


    description = """Mageia GNOME commands."""
    epilog = """Report bugs to Olav Vitters"""
    parser = argparse.ArgumentParser(description=description, epilog=epilog)
    parser.add_argument("-l", "--limit", type=argparse.FileType('r', 0),
                        dest="limit_upstream", metavar="FILE",
                        help="File containing upstream names")
    parser.add_argument("-p", "--root", type=Path, action="store", dest="PKGROOT",
                        help="Package root directory")
    parser.add_argument("-d", "--distro", action="store", dest="distro",
                        help="Distribution release")
    parser.add_argument("--debug", action="store_true", dest="debug",
                        help="Use for debugging")
    parser.set_defaults(
        debug=False, PKGROOT=Downstream.PKGROOT
    )

    # SUBPARSERS
    subparsers = parser.add_subparsers(title='subcommands')
    subparser = subparsers.add_parser('check-latest', help='check for latest version of packages')
    subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
                           help="Increase version for stable upgrades and submit")
    subparser.set_defaults(
        func=cmd_check_latest, submit=False
    )

    subparser = subparsers.add_parser('check-prep', help='check prep phase')
    subparser.add_argument("package", help="Package name")
    subparser.set_defaults(
        func=cmd_check_prep
    )

    subparser = subparsers.add_parser('check-spec', help='check if spec file is ok')
    subparser.add_argument("package", help="Package name", nargs='*')
    subparser.add_argument("-a", "--all", action="store_true", dest="all",
                           help="checkout all Downstream packages")
    subparser.set_defaults(
        func=cmd_check_spec
    )

    subparser = subparsers.add_parser('check-version', help='check if spec version and downstream version match')
    subparser.set_defaults(
        func=cmd_check_version
    )

    subparser = subparsers.add_parser('clean-spec', help='clean specfile')
    subparser.add_argument("package", help="Package name", nargs='*')
    subparser.add_argument("-d", "-s", action="store_true", dest="doit", help="submit the changes")
    subparser.add_argument("-f", "--force", action="store_true")
    subparser.add_argument("-a", "--all", action="store_true", dest="all",
                           help="checkout all Downstream packages")
    subparser.add_argument("--convert", action="store_true", dest="convert_br",
                           help="convert -buildrequirements to perl/pkgconfig if possible")
    subparser.set_defaults(
        func=cmd_clean_spec, doit=False, convert_br=False, all=False, force=False
    )

    subparser = subparsers.add_parser('cleanup', help='cleanup pkg directory')
    subparser.set_defaults(
        func=cmd_cleanup
    )

    subparser = subparsers.add_parser('co', help='checkout all GNOME packages')
    subparser.add_argument("-a", "--all", action="store_true", dest="all",
                           help="checkout all Downstream packages")
    subparser.add_argument("-s", action="store_true", dest="spec_only",
                           help="only checkout SPECS/ directory")
    subparser.add_argument("package", help="Package name", nargs='*')
    subparser.set_defaults(
        func=cmd_checkout, all=False
    )

    subparser = subparsers.add_parser('gnome-release-email',
                                      help='submit packages based on GNOME ftp-release-list email')
    subparser.add_argument("-m", "--mail", help="Email address to send the progress to")
    subparser.add_argument("--fork", action="store_true",
                           help="Fork as quickly as possible")
    subparser.add_argument("-w", "--wait", action="store_true",
                           help="Wait before trying to retrieve the new version")
    subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
                           help="Commit changes and submit")
    subparser.add_argument("-f", "--force", action="store_true",
                           help="Force submission")
    subparser.set_defaults(
        func=cmd_parse_ftp_release_list, force=False, wait=False, fork=False
    )

    subparser = subparsers.add_parser('group-owner', help='list packages by group')
    subparser.add_argument('group', metavar="GROUP", nargs='+')
    subparser.set_defaults(
        func=cmd_group_owner
    )

    subparser = subparsers.add_parser('increase', help='increase version number')
    subparser.add_argument("package", help="Package name")
    subparser.add_argument("version", help="Version number")
    subparser.add_argument("-f", "--force", action="store_true", dest="force",
                           help="Override warnings, just do it")
    subparser.add_argument("-u", "--upstream", action="store_true", dest="upstream",
                           help="Package name reflects the upstream name")
    subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
                           help="Commit changes and submit")
    subparser.add_argument("--no-submit", action="store_false", dest="submit",
                           help="Do not commit changes and submit")
    subparser.add_argument("-a", "--algorithm", choices=hashlib.algorithms_available, dest="algo",
                           help="Hash algorithm")
    subparser.add_argument("--hash", dest="hexdigest",
                           help="Hexdigest of the hash")
    subparser.set_defaults(
        func=cmd_package_new_version, submit=argparse.SUPPRESS, upstream=False,
        hexdigest=None, algo="sha256", force=False
    )

    subparser = subparsers.add_parser('packages', help='list all GNOME packages')
    subparser.add_argument("-m", "--m", action="store_true", dest="upstream",
                           help="Show upstream module")
    subparser.add_argument("--version", action="store_true", dest="show_version",
                           help="Show version numbers")
    subparser.add_argument("--diff", action="store_true", dest="diff",
                           help="Only show packages with different version")
    subparser.add_argument("--sort", type=argparse.FileType('r', 0),
                           dest="sort", metavar="FILE",
                           help="Sort packages according to order in given FILE")
    subparser.add_argument("--spec", action="store_true", dest="spec",
                           help="Give spec file location")
    subparser.set_defaults(
        func=cmd_ls, upstream=False, show_version=False, diff=False
    )

    subparser = subparsers.add_parser('patches', help='list all GNOME patches')
    subparser.add_argument("-p", "--path", action="store_true", dest="path",
                           help="Show full path to patch")
    subparser.set_defaults(
        func=cmd_patches, path=False
    )

    subparser = subparsers.add_parser('rebuild', help='increase release')
    subparser.add_argument("package", help="Package name", nargs="*")
    subparser.add_argument("-m", "--reason", dest="reason", required=True, help="Reason for the rebuild")
    subparser.add_argument("-f", "--force", action="store_true", dest="force",
                           help="Override warnings, just do it")
    subparser.add_argument("-u", "--upstream", action="store_true", dest="upstream",
                           help="Package name reflects the upstream name")
    subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
                           help="Commit changes and submit")
    subparser.add_argument("--no-submit", action="store_false", dest="submit",
                           help="Do not commit changes and submit")
    subparser.set_defaults(
        func=cmd_new_release, submit=argparse.SUPPRESS, upstream=False,
        force=False
    )

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(2)

    options = parser.parse_args()
    if options.limit_upstream:
        Upstream.limit = set(options.limit_upstream.read().strip("\n").splitlines())

    if not hasattr(options, 'submit'):
        options.submit = not options.distro

    Downstream.PKGROOT = options.PKGROOT
    if options.distro:
        Downstream.PKGROOT = options.PKGROOT.joinpath(options.distro)
        Downstream.MEDIA = "Core Release {0} Source,Core {0} Updates Source," \
                "Core {0} Updates Testing Source".format(options.distro)
        Downstream.DISTRO = options.distro

    log_format = '%(levelname): %(message)s'
    logging.basicConfig(format=log_format)
    log = logging.getLogger(None)
    if options.debug:
        log.setLevel(logging.DEBUG)
    else:
        log.setLevel(logging.INFO)


    try:
        options.func(options)
    except KeyboardInterrupt:
        print('Interrupted')
        sys.exit(1)
    except EOFError:
        print('EOF')
        sys.exit(1)
    except IOError as exc:
        if exc.errno != errno.EPIPE:
            raise
        sys.exit(0)

if __name__ == "__main__":
    os.environ['PYTHONUNBUFFERED'] = '1'
    main()