diff options
Diffstat (limited to 'modules/gitmirror/files/on-the-pull')
| -rwxr-xr-x | modules/gitmirror/files/on-the-pull | 359 |
1 files changed, 359 insertions, 0 deletions
diff --git a/modules/gitmirror/files/on-the-pull b/modules/gitmirror/files/on-the-pull new file mode 100755 index 00000000..7d3ede29 --- /dev/null +++ b/modules/gitmirror/files/on-the-pull @@ -0,0 +1,359 @@ +#!/usr/bin/python3 + +import cgi +import http.server +import os +import pwd +import re +import subprocess +import sys +from optparse import OptionParser +from queue import Queue +from threading import Thread + + +GitUpdaterQueue = Queue(0) + + +# NB The following class and bits for running git commands were "liberated" +# from git_multimail.py + +class CommandError(Exception): + def __init__(self, cmd, retcode): + self.cmd = cmd + self.retcode = retcode + Exception.__init__( + self, + 'Command "%s" failed with retcode %s' % (' '.join(cmd), retcode,) + ) + + +# It is assumed in many places that the encoding is uniformly UTF-8, +# so changing these constants is unsupported. But define them here +# anyway, to make it easier to find (at least most of) the places +# where the encoding is important. +ENCODING = 'UTF-8' + + +# The "git" program (this could be changed to include a full path): +GIT_EXECUTABLE = 'git' + + +# How "git" should be invoked (including global arguments), as a list +# of words. This variable is usually initialized automatically by +# read_git_output() via choose_git_command(), but if a value is set +# here then it will be used unconditionally. +GIT_CMD = None + + +def choose_git_command(): + """Decide how to invoke git, and record the choice in GIT_CMD.""" + + global GIT_CMD + + if GIT_CMD is None: + try: + # Check to see whether the "-c" option is accepted (it was + # only added in Git 1.7.2). We don't actually use the + # output of "git --version", though if we needed more + # specific version information this would be the place to + # do it. + cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version'] + read_output(cmd) + GIT_CMD = [GIT_EXECUTABLE, '-c', f'i18n.logoutputencoding={ENCODING}'] + except CommandError: + GIT_CMD = [GIT_EXECUTABLE] + + +def read_git_output(args, inp=None, keepends=False, **kw): + """Read the output of a Git command.""" + + if GIT_CMD is None: + choose_git_command() + + return read_output(GIT_CMD + args, inp=inp, keepends=keepends, **kw) + + +# NOTE: output is in bytes, not a string +def read_output(cmd, inp=None, keepends=False, **kw): + if inp: + stdin = subprocess.PIPE + else: + stdin = None + p = subprocess.Popen( + cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw + ) + (out, err) = p.communicate(inp) + retcode = p.wait() + if retcode: + raise CommandError(cmd, retcode) + if not keepends: + out = out.rstrip(b'\n\r') + return out + + +def run_git_command(args, **kw): + """Runs a git command, ignoring the output. + """ + + read_git_output(args, **kw) + + +def run_command(args, **kw): + """Runs a git command, ignoring the output. + """ + + read_output(args, **kw) + + +class GitUpdater(Thread): + def __init__(self, server, basedir, repoprefix, branch='master', cmd=''): + Thread.__init__(self) + self.server = server + self.basedir = basedir + self.repoprefix = repoprefix + self.branch = branch + self.cmd = cmd + + def run(self): + while 42: + repo = GitUpdaterQueue.get() + if repo is None: + break + try: + print(f"Got update request for '{repo}'", file=sys.stderr) + clonefolder = os.path.join(self.basedir, repo) + if self.repoprefix: + if not repo.startswith(self.repoprefix): + print(f"Ignoring repo '{repo}' due to invalid prefix", file=sys.stderr) + GitUpdaterQueue.task_done() + continue + clonefolder = os.path.join(self.basedir, repo[len(self.repoprefix):]) + command = [] + treeish = '' + changed = True + if not os.path.exists(clonefolder): + cloneparent = os.path.dirname(clonefolder) + if not os.path.exists(cloneparent): + os.makedirs(cloneparent) + cloneurl = self.server + '/' + repo + command = ['clone'] + if '--mirror' == self.branch: + command.append('--mirror') + command.append(cloneurl) + command.append(clonefolder) + print(f"Cloning repo '{repo}' ('{cloneurl}' -> '{clonefolder}')", file=sys.stderr) + + run_git_command(command) + if not os.path.isdir(clonefolder): + raise Exception(f"Clone folder '{clonefolder}' is not a directory. Cloning failed or file in it's place?") + os.chdir(clonefolder) + if '--mirror' != self.branch and 'master' != self.branch: + command = ['checkout', '-t', 'origin/' + self.branch] + run_git_command(command) + elif os.path.isdir(clonefolder): + os.chdir(clonefolder) + print(f"Updating existing repo '{repo}' ({clonefolder})", file=sys.stderr) + command = ['remote', 'update'] + run_git_command(command) + if '--mirror' != self.branch: + sha1before = read_git_output(['rev-parse', 'refs/heads/' + self.branch]) + sha1after = read_git_output(['rev-parse', 'refs/remotes/origin/' + self.branch]) + if sha1before and sha1after: + if sha1before == sha1after: + changed = False + print(f"Repo '{repo}' update on branch '{self.branch}': No changed detected", file=sys.stderr) + else: + treeish = sha1before.decode(ENCODING) + '..' + sha1after.decode(ENCODING) + print(f"Repo '{repo}' update on branch '{self.branch}': Treeish '{treeish}'", file=sys.stderr) + else: + print(f"Repo '{repo}' update on branch '{self.branch}': Before or after sha1 could not be extracted.", file=sys.stderr) + command = ['update-ref', 'refs/heads/' + self.branch, 'refs/remotes/origin/' + self.branch] + run_git_command(command) + command = ['checkout', '-f', self.branch] + run_git_command(command) + else: + raise Exception(f"Clone folder '{clonefolder}' appears to be a file :s") + + if changed and self.cmd: + # Udate the info/web/last-modified file as used by cgit + os.chdir(clonefolder) + command = [self.cmd, repo] + if treeish: + command += [treeish] + run_command(command) + + print(f"Update for '{repo}' complete.", file=sys.stderr) + except Exception as e: + print(f"Error processing repo '{repo}'", file=sys.stderr) + print(str(e), file=sys.stderr) + + GitUpdaterQueue.task_done() + + +class TimeoutServer(http.server.HTTPServer): + def get_request(self): + result = self.socket.accept() + result[0].settimeout(10) + return result + + +class PostHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + ctype, pdict = cgi.parse_header(self.headers['content-type']) + repo = "" + try: + if ctype != 'x-git/repo': + self.send_response(415) + self.end_headers() + return + + # chunked mode is a legitimate reason there would be no content-length, + # but it's easier to just insist on it + length = int(self.headers['content-length']) if self.headers['content-length'] else 0 + if length < 1: + self.send_response(411) + self.end_headers() + return + if length > 1024: + self.send_response(413) + self.end_headers() + return + repo = self.rfile.read(length).decode(ENCODING) + + if re.match(r"^[-_/a-zA-Z0-9\+\.]+$", repo) is None: + self.send_response(400) + self.end_headers() + return + + GitUpdaterQueue.put(repo) + self.send_response(202) + self.end_headers() + + except Exception as e: + print("Error processing request", file=sys.stderr) + print(str(e), file=sys.stderr) + self.send_response(500) + self.end_headers() + + +def Demote(pidfile, uid, gid): + def result(): + piddir = os.path.dirname(pidfile) + if not os.path.exists(piddir): + os.makedirs(piddir) + fd = open(pidfile, 'w') + fd.write(str(os.getpid())) + fd.close() + + if uid and gid: + os.setgid(gid) + os.setuid(uid) + return result + + +def daemonise(options, serverprefix, basefolder): + pw = None + uid = False + gid = False + if options.user: + pw = pwd.getpwnam(options.user) + uid = pw.pw_uid + gid = pw.pw_gid + else: + pw = pwd.getpwnam(os.getlogin()) + + user = pw.pw_name + dirname = pw.pw_dir + env = { + 'HOME': dirname, + 'LOGNAME': user, + 'PWD': dirname, + 'USER': user, + } + if os.getenv('PATH') is not None: + env['PATH'] = os.getenv('PATH') + if os.getenv('PYTHONPATH') is not None: + env['PYTHONPATH'] = os.getenv('PYTHONPATH') + + args = [os.path.abspath(sys.argv[0])] + args.append('-a') + args.append(options.addr) + args.append('-p') + args.append(str(options.port)) + args.append('-r') + args.append(options.repoprefix) + args.append('-b') + args.append(options.branch) + args.append('-c') + args.append(options.cmd) + args.append(serverprefix) + args.append(basefolder) + + subprocess.Popen( + args, preexec_fn=Demote(options.pidfile, uid, gid), cwd=dirname, env=env + ) + exit(0) + + +def main(): + usage = "usage: %prog [options] <serverprefix> <basefolder>" + description = """Listen for repository names being posted via a simple HTTP interface and clone/update them. +POST data simply via curl: +e.g. curl --header 'Content-Type: x-git/repo' --data 'my/repo/name' http://localhost:8000 +""" + parser = OptionParser(usage=usage, description=description) + parser.add_option("-a", "--addr", + type="string", dest="addr", default="0.0.0.0", + help="The interface address to bind to") + parser.add_option("-p", "--port", + type="int", dest="port", default=8000, + help="The port to bind to") + parser.add_option("-r", "--repo-prefix", + type="string", dest="repoprefix", default="", + help="Only handle repositories with the following prefix. This SHOULD contain a trailing slash if it's a folder but SHOULD NOT include a leading slash") + parser.add_option("-b", "--branch", + type="string", dest="branch", default="--mirror", + help="The branch to track on clone. If you pass '--mirror' (the default) as the branch name we will clone as a bare mirror") + parser.add_option("-c", "--cmd", + type="string", dest="cmd", default="", + help="Third party command to exectue after updates. It will execute in the " + "folder of the repo and if we're not in mirror mode, a treeish will be " + "passed as the only argument containing the refs that changed otherwise " + "the command will be run without any arguments") + parser.add_option("-d", "--pid-file", + type="string", dest="pidfile", default="", + help="Daemonise and write pidfile") + parser.add_option("-u", "--user", + type="string", dest="user", default="", + help="Drop privileges to the given user (must be run as root)") + + (options, args) = parser.parse_args() + if len(args) < 2: + parser.error("Both the <serverprefix> and <basefolder> arguments must be supplied.") + if len(args) > 2: + parser.print_usage() + exit(1) + + serverprefix = args[0] + basefolder = args[1] + + if options.pidfile: + daemonise(options, serverprefix, basefolder) + + if options.user: + parser.error("You can only specify a user if you're also deamonising (with a pid file).") + + print("Server started", file=sys.stderr) + srvr = TimeoutServer((options.addr, options.port), PostHandler) + GitUpdater(serverprefix, basefolder, options.repoprefix, options.branch, options.cmd).start() + + try: + srvr.serve_forever() + except KeyboardInterrupt: + GitUpdaterQueue.put(None) + srvr.socket.close() + + +if __name__ == "__main__": + main() |