From 41308f873e8f11b610e3e02042acfdaec84136a1 Mon Sep 17 00:00:00 2001 From: Dan Fandrich Date: Sat, 13 Jan 2024 02:32:14 -0800 Subject: Convert git-post-receive-hook to Python3 This depends on a newer version of pybugz that's Python 3 compatible (tested with 0.14) and git_multimail.py, which has already been updated. Replace token support with API key support, as per the latest pybugz (and Bugzilla). If an API key is found, it will be used and if not found it will fall back to username/password. No attempt is made to try to create an API key in the same way that a token was minted before. Use a different file name for an API key for coexistence with a token, which is still used by other programs. Add a debug flag for enabling more logging to better see when things go wrong. Create variables for configuration items. Log a message when an i18n e-mail is sent. Do a few little code cleanups. --- deployment/mgagit/files/git_multimail.py | 2 +- deployment/mgagit/manifests/init.pp | 2 +- deployment/mgagit/templates/git-post-receive-hook | 198 +++++++++++----------- 3 files changed, 105 insertions(+), 97 deletions(-) diff --git a/deployment/mgagit/files/git_multimail.py b/deployment/mgagit/files/git_multimail.py index 01acf6da..39aa1458 100644 --- a/deployment/mgagit/files/git_multimail.py +++ b/deployment/mgagit/files/git_multimail.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python2 +#! /usr/bin/env python3 __version__ = '1.7.dev' diff --git a/deployment/mgagit/manifests/init.pp b/deployment/mgagit/manifests/init.pp index 2ad9cd64..8aa621ef 100644 --- a/deployment/mgagit/manifests/init.pp +++ b/deployment/mgagit/manifests/init.pp @@ -19,7 +19,7 @@ class mgagit( $reposconf_dir = "${git_homedir}/repos-config" $vhostdir = "${git_homedir}/www" - package { ['mgagit', 'gitolite', 'python-bugz']: + package { ['mgagit', 'gitolite', 'python3-bugz']: ensure => installed, } diff --git a/deployment/mgagit/templates/git-post-receive-hook b/deployment/mgagit/templates/git-post-receive-hook index 087f9133..f8573dd4 100755 --- a/deployment/mgagit/templates/git-post-receive-hook +++ b/deployment/mgagit/templates/git-post-receive-hook @@ -1,18 +1,21 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 +import configparser import os import re import sys +import urllib.error +import urllib.parse +import urllib.request +import xmlrpc.client +from dataclasses import dataclass LIBDIR = '<%= @gitolite_commonhooksdir %>' sys.path.insert(0, LIBDIR) -import git_multimail - -import xmlrpclib -from bugz.bugzilla import BugzillaProxy +import bugz.settings -import urllib2 +import git_multimail # When editing this list, remember to edit the same list in # modules/cgit/templates/filter.commit-links.sh @@ -30,8 +33,17 @@ COMMIT_REPLACE = 'https://gitweb.mageia.org/%s/commit/?id=%s' MAGEIA_BUGZILLA_URL = 'https://bugs.mageia.org/xmlrpc.cgi' MAGEIA_BUGZILLA_PASSWORD_FILE = '.gitzilla-password' -MAGEIA_BUGZILLA_AUTHTOKEN_FILE = '.gitzilla-authtoken' +MAGEIA_BUGZILLA_APIKEY_FILE = '.gitzilla-apikey' # this holds a Bugzilla API key +GITWEB_UPDATE_URL = 'https://gitweb.mageia.org:8000' + +# Bugzilla user +USER_LOGIN = 'bot' + +# Recipient of i18n notifications +I18N_MAIL = 'i18n-reports@ml.mageia.org' +# Set to 1..3 for debug logging (WARNING: this will show passwords to git committers!) +DEBUG = 0 git_multimail.FOOTER_TEMPLATE = """\ @@ -61,6 +73,14 @@ Auto-Submitted: auto-generated REPO_NAME_RE = re.compile(r'^/git/(?P.+?)(?:\.git)?$') + + +# Log a debug message when logging is enabled +def debug(s, *a): + if DEBUG > 0: + print(s % a) + + def repo_shortname(): basename = os.path.abspath(git_multimail.get_git_dir()) m = REPO_NAME_RE.match(basename) @@ -75,64 +95,55 @@ def repo_shortname(): class MageiaEnvironment(git_multimail.Environment): def get_repo_shortname(self): return repo_shortname() + + git_multimail.Environment = MageiaEnvironment # Override the Revision class to inject gitweb/cgit links and any referenced # bug URLs class MageiaLinksRevision(git_multimail.Revision): - bz = None - tokenfile = None - token = None + def __init__(self, reference_change, rev, num, tot): + super().__init__(reference_change, rev, num, tot) + self.bz = None def bugzilla_init(self): - if self.bz is None: - self.tokenfile = os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_AUTHTOKEN_FILE) + if not self.bz: + tokenfile = os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_APIKEY_FILE) + token = None + try: + token = open(tokenfile, 'r').readline().rstrip() + except IOError: + # Assume username/password will be used instead + pass + + passwordfile = os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_PASSWORD_FILE) + pword = None try: - token = open(self.tokenfile, 'r').readline().rstrip() - if token: - self.token = token + pword = open(passwordfile, 'r').readline().rstrip() except IOError: + print('Error: no Bugzilla credentials available; trying anyway') + # There's no real point in continuing, but why not + + class ConfigSettings: pass - self.bz = BugzillaProxy(MAGEIA_BUGZILLA_URL) - return self.bz - - def bugzilla_login(self): - params = { - 'login': 'bot', - 'password': open(os.path.join(os.environ['HOME'], MAGEIA_BUGZILLA_PASSWORD_FILE), 'r').readline().rstrip(), - 'remember': True - } - result = self.bz.User.login(params) - if 'token' in result: - self.token = result['token'] - if self.tokenfile is not None: - fd = open(self.tokenfile, 'w') - fd.write(self.token) - fd.write('\n') - fd.close() - os.chmod(self.tokenfile, 0600) - return True - return False - - def bugzilla_call(self, method, *args): - """Attempt to call method with args. Log in if authentication is required. - """ - try: - if self.token is not None: - args[0]['token'] = self.token - return method(*args) - except xmlrpclib.Fault, fault: - # Fault code 410 means login required - if fault.faultCode == 410 and self.bugzilla_login(): - args[0]['token'] = self.token - return method(*args) - raise + cfg = ConfigSettings() + cfg.connection = 'Mageia' + if token: + # If an API key is found, that will be used instead of user/password + cfg.key = token + cfg.user = USER_LOGIN + cfg.password = pword + cfg.base = MAGEIA_BUGZILLA_URL + cfg.debug = DEBUG + + cfile = configparser.ConfigParser() + cfile.add_section(cfg.connection) + self.bz = bugz.settings.Settings(cfg, cfile) def generate_email_body(self, push): """Show this revision.""" - output = git_multimail.read_git_lines( ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1], keepends=True, @@ -140,6 +151,9 @@ class MageiaLinksRevision(git_multimail.Revision): bugs = {} commit = None idx = 0 + # Modify the mail output on-the-fly to add links; this is sensitive to + # the mail format produced by git_multimail. Also, update Mageia + # Bugzilla if a bug reference is found. for line in output: idx += 1 if line == "---\n": @@ -160,10 +174,10 @@ class MageiaLinksRevision(git_multimail.Revision): output.insert(idx, "\n") idx += 1 - # Attempt to modify bugzilla + # Attempt to modify Bugzilla if "Mageia" in bugs: try: - bz = self.bugzilla_init() + self.bugzilla_init() # Mask email address comment = None @@ -179,46 +193,38 @@ class MageiaLinksRevision(git_multimail.Revision): params = {} params['ids'] = bugs['Mageia'] params['comment'] = { 'body': comment } - self.bugzilla_call(bz.Bug.update, params) - print "Updated bugzilla bugs: %s" % ", ".join(bugs['Mageia']) + self.bz.call_bz(self.bz.bz.Bug.update, params) + print("Updated bugzilla bugs: %s" % ", ".join(bugs['Mageia'])) except: - print "Unable to post to bugzilla bugs: %s :(" % ", ".join(bugs['Mageia']) - print sys.exc_info()[1] + print("Unable to post to bugzilla bugs: %s :(" % ", ".join(bugs['Mageia'])) + print(sys.exc_info()[1]) break + m = COMMIT_RE.search(line) if m: commit = m.group(1) for tracker in BUG_REFS.keys(): foundbugs = BUG_REFS[tracker]['re'].findall(line) - if len(foundbugs): - if not tracker in bugs: + if foundbugs: + if tracker not in bugs: bugs[tracker] = foundbugs else: bugs[tracker] = list(set(bugs[tracker] + foundbugs)) return output + # Override the Revision class to inject gitweb/cgit links and any referenced # bug URLs class MageiaI18NRevision(git_multimail.Revision): """A Change consisting of a single git commit.""" def __init__(self, reference_change, rev, num, tot): - git_multimail.Change.__init__(self, reference_change.environment) - self.reference_change = reference_change - self.rev = rev - self.change_type = self.reference_change.change_type - self.refname = self.reference_change.refname - self.num = num - self.tot = tot - self.author = git_multimail.read_git_output(['log', '--no-walk', '--format=%aN <%%aE>', self.rev.sha1]) - self.recipients = False - self.output = [] - - # -s is short for --no-patch, but -s works on older git's (e.g. 1.7) - self.parents = git_multimail.read_git_lines(['show', '-s', '--format=%P', self.rev.sha1])[0].split() + super().__init__(reference_change, rev, num, tot) + # Don't send to any of the normal recipients + self.recipients = False self.cc_recipients = '' i18n_folders = [] @@ -228,14 +234,16 @@ class MageiaI18NRevision(git_multimail.Revision): if name.endswith("/.tx"): i18n_folders.append(os.path.dirname(name)) - if len(i18n_folders): + if i18n_folders: self.output = git_multimail.read_git_lines( ['log', '-C', '--stat', '-p', '--no-walk', self.rev.sha1, '--'] + i18n_folders, keepends=True, ) - if len(self.output): + if self.output: # We have some output so let's send the mail... - self.recipients = 'i18n-reports@ml.mageia.org' + self.recipients = I18N_MAIL + print(f'Sending i8n notification to {self.recipients}') + def generate_email_body(self, push): """Show this revision.""" @@ -243,32 +251,29 @@ class MageiaI18NRevision(git_multimail.Revision): return self.output - -if __name__ == '__main__': +def main(): # Attempt to write a last-updated file for cgit cosmetics + git_dir = git_multimail.get_git_dir() + infowebdir = os.path.join(git_dir, 'info', 'web') + lastupdated = git_multimail.read_git_output( + ['for-each-ref', '--sort=-committerdate', "--format=%(committerdate:iso8601)", '--count=1', 'refs/heads'], + ) try: - git_dir = git_multimail.get_git_dir() - infowebdir = os.path.join(git_dir, 'info', 'web') if not os.path.exists(infowebdir): os.makedirs(infowebdir) - lastupdated = git_multimail.read_git_output( - ['for-each-ref', '--sort=-committerdate', "--format=%(committerdate:iso8601)", '--count=1', 'refs/heads'], - ) - modfile = open(os.path.join(infowebdir, 'last-modified'), 'w') - modfile.write(lastupdated) - modfile.close() + with open(os.path.join(infowebdir, 'last-modified'), 'w') as modfile: + modfile.write(lastupdated) except Exception: - pass + debug('Warning: could not update git last-modified date: %s', sys.exc_info()[1]) + # Contact the on-the-pull service on the gitweb server with the updated repo try: - req = urllib2.Request('https://gitweb.mageia.org:8000', repo_shortname() + '.git') + req = urllib.request.Request(GITWEB_UPDATE_URL, (repo_shortname() + '.git').encode('utf-8')) req.add_header('Content-Type', 'x-git/repo') - fp = urllib2.urlopen(req, timeout=5) - if (fp): - fp.close() + fp = urllib.request.urlopen(req, timeout=5) + fp.close() except Exception: - pass - + debug('Warning: could not contact gitweb server: %s', sys.exc_info()[1]) config = git_multimail.Config('multimailhook') @@ -278,7 +283,7 @@ if __name__ == '__main__': ) mailer = git_multimail.choose_mailer(config, environment) - # For testing... + # For testing...send mail to stdout only #mailer = git_multimail.OutputMailer(sys.stdout) changes = [] @@ -289,11 +294,10 @@ if __name__ == '__main__': ) push = git_multimail.Push(environment, changes) - # First pass - regular commit mails git_multimail.Revision = MageiaLinksRevision push.send_emails(mailer, body_filter=environment.filter_body) - + # Second pass - i18n commit mails git_multimail.REVISION_HEADER_TEMPLATE = I18N_REVISION_HEADER_TEMPLATE git_multimail.Revision = MageiaI18NRevision @@ -302,5 +306,9 @@ if __name__ == '__main__': change.recipients = False push.send_emails(mailer, body_filter=environment.filter_body) - except git_multimail.ConfigurationException, e: + except git_multimail.ConfigurationException as e: sys.exit(str(e)) + + +if __name__ == '__main__': + main() -- cgit v1.2.1