Convert git-post-receive-hook to Python3

This depends on a newer version of pybugz that's Python 3 compatible
(tested with 0.14) and git_multimail.py, which has already been updated.

Replace token support with API key support, as per the latest pybugz
(and Bugzilla). If an API key is found, it will be used and if not found
it will fall back to username/password. No attempt is made to try to
create an API key in the same way that a token was minted before.  Use a
different file name for an API key for coexistence with a token, which
is still used by other programs.

Add a debug flag for enabling more logging to better see when things go
wrong.

Create variables for configuration items.

Log a message when an i18n e-mail is sent.

Do a few little code cleanups.

3 files changed, 105 insertions, 97 deletions

diff --git a/deployment/mgagit/files/git_multimail.py b/deployment/mgagit/files/git_multimail.py
index 01acf6da..39aa1458 100644
--- a/deployment/mgagit/files/git_multimail.py
+++ b/deployment/mgagit/files/git_multimail.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python2
+#! /usr/bin/env python3
 
 __version__ = '1.7.dev'
 
diff --git a/deployment/mgagit/manifests/init.pp b/deployment/mgagit/manifests/init.pp
index 2ad9cd64..8aa621ef 100644
--- a/deployment/mgagit/manifests/init.pp
+++ b/deployment/mgagit/manifests/init.pp
@@ -19,7 +19,7 @@ class mgagit(
   $reposconf_dir = "${git_homedir}/repos-config"
   $vhostdir = "${git_homedir}/www"
 
-  package { ['mgagit', 'gitolite', 'python-bugz']:
+  package { ['mgagit', 'gitolite', 'python3-bugz']:
     ensure => installed,
   }
 
diff --git a/deployment/mgagit/templates/git-post-receive-hook b/deployment/mgagit/templates/git-post-receive-hook
index 087f9133..f8573dd4 100755
--- a/deployment/mgagit/templates/git-post-receive-hook
+++ b/deployment/mgagit/templates/git-post-receive-hook
@@ -1,18 +1,21 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
+import configparser
 import os
 import re
 import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+import xmlrpc.client
+from dataclasses import dataclass
 
 LIBDIR = '<%= @gitolite_commonhooksdir %>'
 sys.path.insert(0, LIBDIR)
 
-import git_multimail
-
-import xmlrpclib
-from bugz.bugzilla import BugzillaProxy
+import bugz.settings
 
-import urllib2
+import git_multimail
 
 # When editing this list, remember to edit the same list in
 # modules/cgit/templates/filter.commit-links.sh
@@ -30,8 +33,17 @@ COMMIT_REPLACE = 'https://gitweb.mageia.org/%s/commit/?id=%s'
 
 MAGEIA_BUGZILLA_URL = 'https://bugs.mageia.org/xmlrpc.cgi'
 MAGEIA_BUGZILLA_PASSWORD_FILE = '.gitzilla-password'
-MAGEIA_BUGZILLA_AUTHTOKEN_FILE = '.gitzilla-authtoken'
+MAGEIA_BUGZILLA_APIKEY_FILE = '.gitzilla-apikey'  # this holds a Bugzilla API key
+GITWEB_UPDATE_URL = 'https://gitweb.mageia.org:8000'
+
+# Bugzilla user
+USER_LOGIN = 'bot'
+
+# Recipient of i18n notifications
+I18N_MAIL = 'i18n-reports@ml.mageia.org'
 
+# Set to 1..3 for debug logging (WARNING: this will show passwords to git committers!)
+DEBUG = 0
 
 git_multimail.FOOTER_TEMPLATE = """\
 
@@ -61,6 +73,14 @@ Auto-Submitted: auto-generated
 
 
 REPO_NAME_RE = re.compile(r'^/git/(?P<name>.+?)(?:\.git)?$')
+
+
+# Log a debug message when logging is enabled
+def debug(s, *a):
+    if DEBUG > 0:
+        print(s % a)
+
+
 def repo_shortname():
     basename = os.path.abspath(git_multimail.get_git_dir())
     m = REPO_NAME_RE.match(basename)
@@ -75,64 +95,55 @@ def repo_shortname():
 class MageiaEnvironment(git_multimail.Environment):
     def get_repo_shortname(self):
         return repo_shortname()
+
+
 git_multimail.Environment = MageiaEnvironment
 
 
 # Override the Revision class to inject gitweb/cgit links and any referenced
 # bug URLs
 class MageiaLinksRevision(git_multimail.Revision):
-    bz = None
-    tokenfile = None
-    token = None
+    def __init__(self, reference_change, rev, num, tot):
+        super().__init__(reference_change, rev, num, tot)
+        self.bz = None
 
     def bugzilla_init(self):
-        if self.bz is None:
-            self.tokenfile = os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_AUTHTOKEN_FILE)
+        if not self.bz:
+            tokenfile = os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_APIKEY_FILE)
+            token = None
+            try:
+                token = open(tokenfile, 'r').readline().rstrip()
+            except IOError:
+                # Assume username/password will be used instead
+                pass
+
+            passwordfile = os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_PASSWORD_FILE)
+            pword = None
             try:
-                token = open(self.tokenfile, 'r').readline().rstrip()
-                if token:
-                    self.token = token
+                pword = open(passwordfile, 'r').readline().rstrip()
             except IOError:
+                print('Error: no Bugzilla credentials available; trying anyway')
+                # There's no real point in continuing, but why not
+
+            class ConfigSettings:
                 pass
 
-            self.bz = BugzillaProxy(MAGEIA_BUGZILLA_URL)
-        return self.bz
-
-    def bugzilla_login(self):
-        params = {
-          'login': 'bot',
-          'password': open(os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_PASSWORD_FILE), 'r').readline().rstrip(),
-          'remember': True
-        }
-        result = self.bz.User.login(params)
-        if 'token' in result:
-            self.token = result['token']
-            if self.tokenfile is not None:
-                fd = open(self.tokenfile, 'w')
-                fd.write(self.token)
-                fd.write('\n')
-                fd.close()
-                os.chmod(self.tokenfile, 0600)
-            return True
-        return False
-
-    def bugzilla_call(self, method, *args):
-        """Attempt to call method with args. Log in if authentication is required.
-        """
-        try:
-            if self.token is not None:
-                args[0]['token'] = self.token
-            return method(*args)
-        except xmlrpclib.Fault, fault:
-            # Fault code 410 means login required
-            if fault.faultCode == 410 and self.bugzilla_login():
-                args[0]['token'] = self.token
-                return method(*args)
-            raise
+            cfg = ConfigSettings()
+            cfg.connection = 'Mageia'
+            if token:
+                # If an API key is found, that will be used instead of user/password
+                cfg.key = token
+            cfg.user = USER_LOGIN
+            cfg.password = pword
+            cfg.base = MAGEIA_BUGZILLA_URL
+            cfg.debug = DEBUG
+
+            cfile = configparser.ConfigParser()
+            cfile.add_section(cfg.connection)
+            self.bz = bugz.settings.Settings(cfg, cfile)
 
     def generate_email_body(self, push):
         """Show this revision."""
-
         output = git_multimail.read_git_lines(
             ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1],
             keepends=True,
@@ -140,6 +151,9 @@ class MageiaLinksRevision(git_multimail.Revision):
         bugs = {}
         commit = None
         idx = 0
+        # Modify the mail output on-the-fly to add links; this is sensitive to
+        # the mail format produced by git_multimail. Also, update Mageia
+        # Bugzilla if a bug reference is found.
         for line in output:
             idx += 1
             if line == "---\n":
@@ -160,10 +174,10 @@ class MageiaLinksRevision(git_multimail.Revision):
                     output.insert(idx, "\n")
                     idx += 1
 
-                    # Attempt to modify bugzilla
+                    # Attempt to modify Bugzilla
                     if "Mageia" in bugs:
                         try:
-                            bz = self.bugzilla_init()
+                            self.bugzilla_init()
 
                             # Mask email address
                             comment = None
@@ -179,46 +193,38 @@ class MageiaLinksRevision(git_multimail.Revision):
                             params = {}
                             params['ids'] = bugs['Mageia']
                             params['comment'] = { 'body': comment }
-                            self.bugzilla_call(bz.Bug.update, params)
-                            print "Updated bugzilla bugs: %s" % ", ".join(bugs['Mageia'])
+                            self.bz.call_bz(self.bz.bz.Bug.update, params)
+                            print("Updated bugzilla bugs: %s" % ", ".join(bugs['Mageia']))
                         except:
-                            print "Unable to post to bugzilla bugs: %s :(" % ", ".join(bugs['Mageia'])
-                            print sys.exc_info()[1]
+                            print("Unable to post to bugzilla bugs: %s :(" % ", ".join(bugs['Mageia']))
+                            print(sys.exc_info()[1])
 
                 break
+
             m = COMMIT_RE.search(line)
             if m:
                 commit = m.group(1)
             for tracker in BUG_REFS.keys():
                 foundbugs = BUG_REFS[tracker]['re'].findall(line)
-                if len(foundbugs):
-                    if not tracker in bugs:
+                if foundbugs:
+                    if tracker not in bugs:
                         bugs[tracker] = foundbugs
                     else:
                         bugs[tracker] = list(set(bugs[tracker] + foundbugs))
 
         return output
 
+
 # Override the Revision class to inject gitweb/cgit links and any referenced
 # bug URLs
 class MageiaI18NRevision(git_multimail.Revision):
     """A Change consisting of a single git commit."""
 
     def __init__(self, reference_change, rev, num, tot):
-        git_multimail.Change.__init__(self, reference_change.environment)
-        self.reference_change = reference_change
-        self.rev = rev
-        self.change_type = self.reference_change.change_type
-        self.refname = self.reference_change.refname
-        self.num = num
-        self.tot = tot
-        self.author = git_multimail.read_git_output(['log', '--no-walk', '--format=%aN <%%aE>', self.rev.sha1])
-        self.recipients = False
-        self.output = []
-
-        # -s is short for --no-patch, but -s works on older git's (e.g. 1.7)
-        self.parents = git_multimail.read_git_lines(['show', '-s', '--format=%P', self.rev.sha1])[0].split()
+        super().__init__(reference_change, rev, num, tot)
 
+        # Don't send to any of the normal recipients
+        self.recipients = False
         self.cc_recipients = ''
 
         i18n_folders = []
@@ -228,14 +234,16 @@ class MageiaI18NRevision(git_multimail.Revision):
             if name.endswith("/.tx"):
                 i18n_folders.append(os.path.dirname(name))
 
-        if len(i18n_folders):
+        if i18n_folders:
             self.output = git_multimail.read_git_lines(
                 ['log', '-C', '--stat', '-p', '--no-walk', self.rev.sha1, '--'] + i18n_folders,
                 keepends=True,
                 )
-            if len(self.output):
+            if self.output:
                 # We have some output so let's send the mail...
-                self.recipients = 'i18n-reports@ml.mageia.org'
+                self.recipients = I18N_MAIL
+                print(f'Sending i8n notification to {self.recipients}')
+
 
     def generate_email_body(self, push):
         """Show this revision."""
@@ -243,32 +251,29 @@ class MageiaI18NRevision(git_multimail.Revision):
         return self.output
 
 
-
-if __name__ == '__main__':
+def main():
    # Attempt to write a last-updated file for cgit cosmetics
+    git_dir = git_multimail.get_git_dir()
+    infowebdir = os.path.join(git_dir, 'info', 'web')
+    lastupdated = git_multimail.read_git_output(
+        ['for-each-ref', '--sort=-committerdate', "--format=%(committerdate:iso8601)", '--count=1', 'refs/heads'],
+    )
     try:
-        git_dir = git_multimail.get_git_dir()
-        infowebdir = os.path.join(git_dir, 'info', 'web')
         if not os.path.exists(infowebdir):
             os.makedirs(infowebdir)
-        lastupdated = git_multimail.read_git_output(
-            ['for-each-ref', '--sort=-committerdate', "--format=%(committerdate:iso8601)", '--count=1', 'refs/heads'],
-        )
-        modfile = open(os.path.join(infowebdir, 'last-modified'), 'w')
-        modfile.write(lastupdated)
-        modfile.close()
+        with open(os.path.join(infowebdir, 'last-modified'), 'w') as modfile:
+            modfile.write(lastupdated)
     except Exception:
-        pass
+        debug('Warning: could not update git last-modified date: %s', sys.exc_info()[1])
 
+    # Contact the on-the-pull service on the gitweb server with the updated repo
     try:
-        req = urllib2.Request('https://gitweb.mageia.org:8000', repo_shortname() + '.git')
+        req = urllib.request.Request(GITWEB_UPDATE_URL, (repo_shortname() + '.git').encode('utf-8'))
         req.add_header('Content-Type', 'x-git/repo')
-        fp = urllib2.urlopen(req, timeout=5)
-        if (fp):
-            fp.close()
+        fp = urllib.request.urlopen(req, timeout=5)
+        fp.close()
     except Exception:
-        pass
-
+        debug('Warning: could not contact gitweb server: %s', sys.exc_info()[1])
 
     config = git_multimail.Config('multimailhook')
 
@@ -278,7 +283,7 @@ if __name__ == '__main__':
             )
 
         mailer = git_multimail.choose_mailer(config, environment)
-        # For testing...
+        # For testing...send mail to stdout only
         #mailer = git_multimail.OutputMailer(sys.stdout)
 
         changes = []
@@ -289,11 +294,10 @@ if __name__ == '__main__':
                 )
         push = git_multimail.Push(environment, changes)
 
-
         # First pass - regular commit mails
         git_multimail.Revision = MageiaLinksRevision
         push.send_emails(mailer, body_filter=environment.filter_body)
- 
+
         # Second pass - i18n commit mails
         git_multimail.REVISION_HEADER_TEMPLATE = I18N_REVISION_HEADER_TEMPLATE
         git_multimail.Revision = MageiaI18NRevision
@@ -302,5 +306,9 @@ if __name__ == '__main__':
             change.recipients = False
         push.send_emails(mailer, body_filter=environment.filter_body)
 
-    except git_multimail.ConfigurationException, e:
+    except git_multimail.ConfigurationException as e:
         sys.exit(str(e))
+
+
+if __name__ == '__main__':
+    main()