Switch gitmirror files from templates to files

There are no template substitutions needed in these files, so allowing
them opens the danger of substitutions happening unknowingly with future
changes to these files.

1 files changed, 359 insertions, 0 deletions

diff --git a/modules/gitmirror/files/on-the-pull b/modules/gitmirror/files/on-the-pull
new file mode 100755
index 00000000..7d3ede29
--- /dev/null
+++ b/modules/gitmirror/files/on-the-pull
@@ -0,0 +1,359 @@
+#!/usr/bin/python3
+
+import cgi
+import http.server
+import os
+import pwd
+import re
+import subprocess
+import sys
+from optparse import OptionParser
+from queue import Queue
+from threading import Thread
+
+
+GitUpdaterQueue = Queue(0)
+
+
+# NB The following class and bits for running git commands were "liberated"
+# from git_multimail.py
+
+class CommandError(Exception):
+    def __init__(self, cmd, retcode):
+        self.cmd = cmd
+        self.retcode = retcode
+        Exception.__init__(
+            self,
+            'Command "%s" failed with retcode %s' % (' '.join(cmd), retcode,)
+            )
+
+
+# It is assumed in many places that the encoding is uniformly UTF-8,
+# so changing these constants is unsupported.  But define them here
+# anyway, to make it easier to find (at least most of) the places
+# where the encoding is important.
+ENCODING = 'UTF-8'
+
+
+# The "git" program (this could be changed to include a full path):
+GIT_EXECUTABLE = 'git'
+
+
+# How "git" should be invoked (including global arguments), as a list
+# of words.  This variable is usually initialized automatically by
+# read_git_output() via choose_git_command(), but if a value is set
+# here then it will be used unconditionally.
+GIT_CMD = None
+
+
+def choose_git_command():
+    """Decide how to invoke git, and record the choice in GIT_CMD."""
+
+    global GIT_CMD
+
+    if GIT_CMD is None:
+        try:
+            # Check to see whether the "-c" option is accepted (it was
+            # only added in Git 1.7.2).  We don't actually use the
+            # output of "git --version", though if we needed more
+            # specific version information this would be the place to
+            # do it.
+            cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version']
+            read_output(cmd)
+            GIT_CMD = [GIT_EXECUTABLE, '-c', f'i18n.logoutputencoding={ENCODING}']
+        except CommandError:
+            GIT_CMD = [GIT_EXECUTABLE]
+
+
+def read_git_output(args, inp=None, keepends=False, **kw):
+    """Read the output of a Git command."""
+
+    if GIT_CMD is None:
+        choose_git_command()
+
+    return read_output(GIT_CMD + args, inp=inp, keepends=keepends, **kw)
+
+
+# NOTE: output is in bytes, not a string
+def read_output(cmd, inp=None, keepends=False, **kw):
+    if inp:
+        stdin = subprocess.PIPE
+    else:
+        stdin = None
+    p = subprocess.Popen(
+        cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw
+        )
+    (out, err) = p.communicate(inp)
+    retcode = p.wait()
+    if retcode:
+        raise CommandError(cmd, retcode)
+    if not keepends:
+        out = out.rstrip(b'\n\r')
+    return out
+
+
+def run_git_command(args, **kw):
+    """Runs a git command, ignoring the output.
+    """
+
+    read_git_output(args, **kw)
+
+
+def run_command(args, **kw):
+    """Runs a git command, ignoring the output.
+    """
+
+    read_output(args, **kw)
+
+
+class GitUpdater(Thread):
+    def __init__(self, server, basedir, repoprefix, branch='master', cmd=''):
+        Thread.__init__(self)
+        self.server = server
+        self.basedir = basedir
+        self.repoprefix = repoprefix
+        self.branch = branch
+        self.cmd = cmd
+
+    def run(self):
+        while 42:
+            repo = GitUpdaterQueue.get()
+            if repo is None:
+                break
+            try:
+                print(f"Got update request for '{repo}'", file=sys.stderr)
+                clonefolder = os.path.join(self.basedir, repo)
+                if self.repoprefix:
+                    if not repo.startswith(self.repoprefix):
+                        print(f"Ignoring repo '{repo}' due to invalid prefix", file=sys.stderr)
+                        GitUpdaterQueue.task_done()
+                        continue
+                    clonefolder = os.path.join(self.basedir, repo[len(self.repoprefix):])
+                command = []
+                treeish = ''
+                changed = True
+                if not os.path.exists(clonefolder):
+                    cloneparent = os.path.dirname(clonefolder)
+                    if not os.path.exists(cloneparent):
+                        os.makedirs(cloneparent)
+                    cloneurl = self.server + '/' + repo
+                    command = ['clone']
+                    if '--mirror' == self.branch:
+                        command.append('--mirror')
+                    command.append(cloneurl)
+                    command.append(clonefolder)
+                    print(f"Cloning repo '{repo}' ('{cloneurl}' -> '{clonefolder}')", file=sys.stderr)
+
+                    run_git_command(command)
+                    if not os.path.isdir(clonefolder):
+                        raise Exception(f"Clone folder '{clonefolder}' is not a directory. Cloning failed or file in it's place?")
+                    os.chdir(clonefolder)
+                    if '--mirror' != self.branch and 'master' != self.branch:
+                        command = ['checkout', '-t', 'origin/' + self.branch]
+                        run_git_command(command)
+                elif os.path.isdir(clonefolder):
+                    os.chdir(clonefolder)
+                    print(f"Updating existing repo '{repo}' ({clonefolder})", file=sys.stderr)
+                    command = ['remote', 'update']
+                    run_git_command(command)
+                    if '--mirror' != self.branch:
+                        sha1before = read_git_output(['rev-parse', 'refs/heads/' + self.branch])
+                        sha1after = read_git_output(['rev-parse', 'refs/remotes/origin/' + self.branch])
+                        if sha1before and sha1after:
+                            if sha1before == sha1after:
+                                changed = False
+                                print(f"Repo '{repo}' update on branch '{self.branch}': No changed detected", file=sys.stderr)
+                            else:
+                                treeish = sha1before.decode(ENCODING) + '..' + sha1after.decode(ENCODING)
+                                print(f"Repo '{repo}' update on branch '{self.branch}': Treeish '{treeish}'", file=sys.stderr)
+                        else:
+                            print(f"Repo '{repo}' update on branch '{self.branch}': Before or after sha1 could not be extracted.", file=sys.stderr)
+                        command = ['update-ref', 'refs/heads/' + self.branch, 'refs/remotes/origin/' + self.branch]
+                        run_git_command(command)
+                        command = ['checkout', '-f', self.branch]
+                        run_git_command(command)
+                else:
+                    raise Exception(f"Clone folder '{clonefolder}' appears to be a file :s")
+
+                if changed and self.cmd:
+                    # Udate the info/web/last-modified file as used by cgit
+                    os.chdir(clonefolder)
+                    command = [self.cmd, repo]
+                    if treeish:
+                        command += [treeish]
+                    run_command(command)
+
+                print(f"Update for '{repo}' complete.", file=sys.stderr)
+            except Exception as e:
+                print(f"Error processing repo '{repo}'", file=sys.stderr)
+                print(str(e), file=sys.stderr)
+
+            GitUpdaterQueue.task_done()
+
+
+class TimeoutServer(http.server.HTTPServer):
+    def get_request(self):
+        result = self.socket.accept()
+        result[0].settimeout(10)
+        return result
+
+
+class PostHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        ctype, pdict = cgi.parse_header(self.headers['content-type'])
+        repo = ""
+        try:
+            if ctype != 'x-git/repo':
+                self.send_response(415)
+                self.end_headers()
+                return
+
+            # chunked mode is a legitimate reason there would be no content-length,
+            # but it's easier to just insist on it
+            length = int(self.headers['content-length']) if self.headers['content-length'] else 0
+            if length < 1:
+                self.send_response(411)
+                self.end_headers()
+                return
+            if length > 1024:
+                self.send_response(413)
+                self.end_headers()
+                return
+            repo = self.rfile.read(length).decode(ENCODING)
+
+            if re.match(r"^[-_/a-zA-Z0-9\+\.]+$", repo) is None:
+                self.send_response(400)
+                self.end_headers()
+                return
+
+            GitUpdaterQueue.put(repo)
+            self.send_response(202)
+            self.end_headers()
+
+        except Exception as e:
+            print("Error processing request", file=sys.stderr)
+            print(str(e), file=sys.stderr)
+            self.send_response(500)
+            self.end_headers()
+
+
+def Demote(pidfile, uid, gid):
+    def result():
+        piddir = os.path.dirname(pidfile)
+        if not os.path.exists(piddir):
+            os.makedirs(piddir)
+        fd = open(pidfile, 'w')
+        fd.write(str(os.getpid()))
+        fd.close()
+
+        if uid and gid:
+            os.setgid(gid)
+            os.setuid(uid)
+    return result
+
+
+def daemonise(options, serverprefix, basefolder):
+    pw = None
+    uid = False
+    gid = False
+    if options.user:
+        pw = pwd.getpwnam(options.user)
+        uid = pw.pw_uid
+        gid = pw.pw_gid
+    else:
+        pw = pwd.getpwnam(os.getlogin())
+
+    user = pw.pw_name
+    dirname = pw.pw_dir
+    env = {
+      'HOME': dirname,
+      'LOGNAME': user,
+      'PWD': dirname,
+      'USER': user,
+    }
+    if os.getenv('PATH') is not None:
+        env['PATH'] = os.getenv('PATH')
+    if os.getenv('PYTHONPATH') is not None:
+        env['PYTHONPATH'] = os.getenv('PYTHONPATH')
+
+    args = [os.path.abspath(sys.argv[0])]
+    args.append('-a')
+    args.append(options.addr)
+    args.append('-p')
+    args.append(str(options.port))
+    args.append('-r')
+    args.append(options.repoprefix)
+    args.append('-b')
+    args.append(options.branch)
+    args.append('-c')
+    args.append(options.cmd)
+    args.append(serverprefix)
+    args.append(basefolder)
+
+    subprocess.Popen(
+        args, preexec_fn=Demote(options.pidfile, uid, gid), cwd=dirname, env=env
+    )
+    exit(0)
+
+
+def main():
+    usage = "usage: %prog [options] <serverprefix> <basefolder>"
+    description = """Listen for repository names being posted via a simple HTTP interface and clone/update them.
+POST data simply via curl:
+e.g. curl --header 'Content-Type: x-git/repo' --data 'my/repo/name' http://localhost:8000
+"""
+    parser = OptionParser(usage=usage, description=description)
+    parser.add_option("-a", "--addr",
+                      type="string", dest="addr", default="0.0.0.0",
+                      help="The interface address to bind to")
+    parser.add_option("-p", "--port",
+                      type="int", dest="port", default=8000,
+                      help="The port to bind to")
+    parser.add_option("-r", "--repo-prefix",
+                      type="string", dest="repoprefix", default="",
+                      help="Only handle repositories with the following prefix. This SHOULD contain a trailing slash if it's a folder but SHOULD NOT include a leading slash")
+    parser.add_option("-b", "--branch",
+                      type="string", dest="branch", default="--mirror",
+                      help="The branch to track on clone. If you pass '--mirror' (the default) as the branch name we will clone as a bare mirror")
+    parser.add_option("-c", "--cmd",
+                      type="string", dest="cmd", default="",
+                      help="Third party command to exectue after updates. It will execute in the "
+                      "folder of the repo and if we're not in mirror mode, a treeish will be "
+                      "passed as the only argument containing the refs that changed otherwise "
+                      "the command will be run without any arguments")
+    parser.add_option("-d", "--pid-file",
+                      type="string", dest="pidfile", default="",
+                      help="Daemonise and write pidfile")
+    parser.add_option("-u", "--user",
+                      type="string", dest="user", default="",
+                      help="Drop privileges to the given user (must be run as root)")
+
+    (options, args) = parser.parse_args()
+    if len(args) < 2:
+        parser.error("Both the <serverprefix> and <basefolder> arguments must be supplied.")
+    if len(args) > 2:
+        parser.print_usage()
+        exit(1)
+
+    serverprefix = args[0]
+    basefolder = args[1]
+
+    if options.pidfile:
+        daemonise(options, serverprefix, basefolder)
+
+    if options.user:
+        parser.error("You can only specify a user if you're also deamonising (with a pid file).")
+
+    print("Server started", file=sys.stderr)
+    srvr = TimeoutServer((options.addr, options.port), PostHandler)
+    GitUpdater(serverprefix, basefolder, options.repoprefix, options.branch, options.cmd).start()
+
+    try:
+        srvr.serve_forever()
+    except KeyboardInterrupt:
+        GitUpdaterQueue.put(None)
+        srvr.socket.close()
+
+
+if __name__ == "__main__":
+    main()