#!/usr/bin/python3 import cgi import http.server import os import pwd import re import subprocess import sys from optparse import OptionParser from queue import Queue from threading import Thread GitUpdaterQueue = Queue(0) # NB The following class and bits for running git commands were "liberated" # from git_multimail.py class CommandError(Exception): def __init__(self, cmd, retcode): self.cmd = cmd self.retcode = retcode Exception.__init__( self, 'Command "%s" failed with retcode %s' % (' '.join(cmd), retcode,) ) # It is assumed in many places that the encoding is uniformly UTF-8, # so changing these constants is unsupported. But define them here # anyway, to make it easier to find (at least most of) the places # where the encoding is important. ENCODING = 'UTF-8' # The "git" program (this could be changed to include a full path): GIT_EXECUTABLE = 'git' # How "git" should be invoked (including global arguments), as a list # of words. This variable is usually initialized automatically by # read_git_output() via choose_git_command(), but if a value is set # here then it will be used unconditionally. GIT_CMD = None def choose_git_command(): """Decide how to invoke git, and record the choice in GIT_CMD.""" global GIT_CMD if GIT_CMD is None: try: # Check to see whether the "-c" option is accepted (it was # only added in Git 1.7.2). We don't actually use the # output of "git --version", though if we needed more # specific version information this would be the place to # do it. cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version'] read_output(cmd) GIT_CMD = [GIT_EXECUTABLE, '-c', f'i18n.logoutputencoding={ENCODING}'] except CommandError: GIT_CMD = [GIT_EXECUTABLE] def read_git_output(args, inp=None, keepends=False, **kw): """Read the output of a Git command.""" if GIT_CMD is None: choose_git_command() return read_output(GIT_CMD + args, inp=inp, keepends=keepends, **kw) # NOTE: output is in bytes, not a string def read_output(cmd, inp=None, keepends=False, **kw): if inp: stdin = subprocess.PIPE else: stdin = None p = subprocess.Popen( cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw ) (out, err) = p.communicate(inp) retcode = p.wait() if retcode: raise CommandError(cmd, retcode) if not keepends: out = out.rstrip(b'\n\r') return out def run_git_command(args, **kw): """Runs a git command, ignoring the output. """ read_git_output(args, **kw) def run_command(args, **kw): """Runs a git command, ignoring the output. """ read_output(args, **kw) class GitUpdater(Thread): def __init__(self, server, basedir, repoprefix, branch='master', cmd=''): Thread.__init__(self) self.server = server self.basedir = basedir self.repoprefix = repoprefix self.branch = branch self.cmd = cmd def run(self): while 42: repo = GitUpdaterQueue.get() if repo is None: break try: print(f"Got update request for '{repo}'", file=sys.stderr) clonefolder = os.path.join(self.basedir, repo) if self.repoprefix: if not repo.startswith(self.repoprefix): print(f"Ignoring repo '{repo}' due to invalid prefix", file=sys.stderr) GitUpdaterQueue.task_done() continue clonefolder = os.path.join(self.basedir, repo[len(self.repoprefix):]) command = [] treeish = '' changed = True if not os.path.exists(clonefolder): cloneparent = os.path.dirname(clonefolder) if not os.path.exists(cloneparent): os.makedirs(cloneparent) cloneurl = self.server + '/' + repo command = ['clone'] if '--mirror' == self.branch: command.append('--mirror') command.append(cloneurl) command.append(clonefolder) print(f"Cloning repo '{repo}' ('{cloneurl}' -> '{clonefolder}')", file=sys.stderr) run_git_command(command) if not os.path.isdir(clonefolder): raise Exception(f"Clone folder '{clonefolder}' is not a directory. Cloning failed or file in it's place?") os.chdir(clonefolder) if '--mirror' != self.branch and 'master' != self.branch: command = ['checkout', '-t', 'origin/' + self.branch] run_git_command(command) elif os.path.isdir(clonefolder): os.chdir(clonefolder) print(f"Updating existing repo '{repo}' ({clonefolder})", file=sys.stderr) command = ['remote', 'update'] run_git_command(command) if '--mirror' != self.branch: sha1before = read_git_output(['rev-parse', 'refs/heads/' + self.branch]) sha1after = read_git_output(['rev-parse', 'refs/remotes/origin/' + self.branch]) if sha1before and sha1after: if sha1before == sha1after: changed = False print(f"Repo '{repo}' update on branch '{self.branch}': No changed detected", file=sys.stderr) else: treeish = sha1before.decode(ENCODING) + '..' + sha1after.decode(ENCODING) print(f"Repo '{repo}' update on branch '{self.branch}': Treeish '{treeish}'", file=sys.stderr) else: print(f"Repo '{repo}' update on branch '{self.branch}': Before or after sha1 could not be extracted.", file=sys.stderr) command = ['update-ref', 'refs/heads/' + self.branch, 'refs/remotes/origin/' + self.branch] run_git_command(command) command = ['checkout', '-f', self.branch] run_git_command(command) else: raise Exception(f"Clone folder '{clonefolder}' appears to be a file :s") if changed and self.cmd: # Update the info/web/last-modified file as used by cgit os.chdir(clonefolder) command = [self.cmd, repo] if treeish: command += [treeish] run_command(command) print(f"Update for '{repo}' complete.", file=sys.stderr) except Exception as e: print(f"Error processing repo '{repo}'", file=sys.stderr) print(str(e), file=sys.stderr) GitUpdaterQueue.task_done() sys.stderr.flush() class TimeoutServer(http.server.HTTPServer): def get_request(self): result = self.socket.accept() result[0].settimeout(10) return result class PostHandler(http.server.BaseHTTPRequestHandler): def do_POST(self): ctype, pdict = cgi.parse_header(self.headers['content-type']) repo = "" try: if ctype != 'x-git/repo': self.send_response(415) self.end_headers() return # chunked mode is a legitimate reason there would be no content-length, # but it's easier to just insist on it length = int(self.headers['content-length']) if self.headers['content-length'] else 0 if length < 1: self.send_response(411) self.end_headers() return if length > 1024: self.send_response(413) self.end_headers() return repo = self.rfile.read(length).decode(ENCODING) if re.match(r"^[-_/a-zA-Z0-9\+\.]+$", repo) is None: self.send_response(400) self.end_headers() return GitUpdaterQueue.put(repo) self.send_response(202) self.end_headers() except Exception as e: print("Error processing request", file=sys.stderr) print(str(e), file=sys.stderr) self.send_response(500) self.end_headers() sys.stderr.flush() def Demote(pidfile, uid, gid): def result(): piddir = os.path.dirname(pidfile) if not os.path.exists(piddir): os.makedirs(piddir) fd = open(pidfile, 'w') fd.write(str(os.getpid())) fd.close() if uid and gid: os.setgid(gid) os.setuid(uid) return result def daemonise(options, serverprefix, basefolder): pw = None uid = False gid = False if options.user: pw = pwd.getpwnam(options.user) uid = pw.pw_uid gid = pw.pw_gid else: pw = pwd.getpwnam(os.getlogin()) user = pw.pw_name dirname = pw.pw_dir env = { 'HOME': dirname, 'LOGNAME': user, 'PWD': dirname, 'USER': user, } if os.getenv('PATH') is not None: env['PATH'] = os.getenv('PATH') if os.getenv('PYTHONPATH') is not None: env['PYTHONPATH'] = os.getenv('PYTHONPATH') args = [os.path.abspath(sys.argv[0])] args.append('-a') args.append(options.addr) args.append('-p') args.append(str(options.port)) args.append('-r') args.append(options.repoprefix) args.append('-b') args.append(options.branch) args.append('-c') args.append(options.cmd) args.append(serverprefix) args.append(basefolder) subprocess.Popen( args, preexec_fn=Demote(options.pidfile, uid, gid), cwd=dirname, env=env ) exit(0) def main(): usage = "usage: %prog [options] " description = """Listen for repository names being posted via a simple HTTP interface and clone/update them. POST data simply via curl: e.g. curl --header 'Content-Type: x-git/repo' --data 'my/repo/name' http://localhost:8000 """ parser = OptionParser(usage=usage, description=description) parser.add_option("-a", "--addr", type="string", dest="addr", default="0.0.0.0", help="The interface address to bind to") parser.add_option("-p", "--port", type="int", dest="port", default=8000, help="The port to bind to") parser.add_option("-r", "--repo-prefix", type="string", dest="repoprefix", default="", help="Only handle repositories with the following prefix. This SHOULD contain a trailing slash if it's a folder but SHOULD NOT include a leading slash") parser.add_option("-b", "--branch", type="string", dest="branch", default="--mirror", help="The branch to track on clone. If you pass '--mirror' (the default) as the branch name we will clone as a bare mirror") parser.add_option("-c", "--cmd", type="string", dest="cmd", default="", help="Third party command to execute after updates. It will execute in the " "folder of the repo and if we're not in mirror mode, a treeish will be " "passed as the only argument containing the refs that changed otherwise " "the command will be run without any arguments") parser.add_option("-d", "--pid-file", type="string", dest="pidfile", default="", help="Daemonise and write pidfile") parser.add_option("-u", "--user", type="string", dest="user", default="", help="Drop privileges to the given user (must be run as root)") (options, args) = parser.parse_args() if len(args) < 2: parser.error("Both the and arguments must be supplied.") if len(args) > 2: parser.print_usage() exit(1) serverprefix = args[0] basefolder = args[1] if options.pidfile: daemonise(options, serverprefix, basefolder) if options.user: parser.error("You can only specify a user if you're also deamonising (with a pid file).") print("Server started", file=sys.stderr) sys.stderr.flush() srvr = TimeoutServer((options.addr, options.port), PostHandler) updater = GitUpdater(serverprefix, basefolder, options.repoprefix, options.branch, options.cmd) updater.start() try: srvr.serve_forever() except KeyboardInterrupt: srvr.socket.close() GitUpdaterQueue.put(None) updater.join() if __name__ == "__main__": main()