Add a new system to ensure our git repos are mirrored properly.

This is a simple python daemon that I wrote which can be 'pinged' and told
to update (or freshly clone) given git repos.

Deploy this script on alamut (not started automatically yet)

5 files changed, 472 insertions, 0 deletions

diff --git a/manifests/nodes/alamut.pp b/manifests/nodes/alamut.pp
index 92d7abfa..c74a2589 100644
--- a/manifests/nodes/alamut.pp
+++ b/manifests/nodes/alamut.pp
@@ -73,6 +73,7 @@ node alamut {
     #Enable back to test.
     include repositories::git_mirror
     include cgit
+    include gitmirror
 
     include xymon::server
     apache::vhost_simple { "xymon.$domain":
diff --git a/modules/gitmirror/manifests/init.pp b/modules/gitmirror/manifests/init.pp
new file mode 100644
index 00000000..4ab44c0a
--- /dev/null
+++ b/modules/gitmirror/manifests/init.pp
@@ -0,0 +1,13 @@
+class gitmirror {
+    mga_common::local_script { 'on-the-pull':
+        content => template('gitmirror/on-the-pull'),
+    }
+    file { '/etc/init.d/on-the-pull':
+        content => template('gitmirror/on-the-pull.init'),
+        mode    => 755,
+    }
+
+    mga_common::local_script { 'gitmirror-sync-metadata':
+        content => template('gitmirror/rsync-metadata.sh'),
+    }
+}
diff --git a/modules/gitmirror/templates/on-the-pull b/modules/gitmirror/templates/on-the-pull
new file mode 100755
index 00000000..1a82785b
--- /dev/null
+++ b/modules/gitmirror/templates/on-the-pull
@@ -0,0 +1,369 @@
+#!/usr/bin/python
+
+import sys
+import os
+import pwd
+import BaseHTTPServer
+import cgi
+import re
+import subprocess
+# For Python 2.4 compatibility, favour optparse
+from optparse import OptionParser
+from time import sleep
+from threading import Thread
+from Queue import Queue
+
+
+class UpdaterQueue(Queue):
+    # Python 2.4 Queue compatibility methods
+
+    def task_done(self):
+        """(Wrapper for Python 2.4 compatibility) Indicate that a formerly enqueued task is complete. Used for join().
+
+        WARNING: Does nothing in Python 2.4 for now
+        """
+        # TODO: Make this do something useful in Python 2.4
+        if hasattr(Queue, 'task_done'):
+            return Queue.task_done(self)
+
+    def join(self):
+        """(Wrapper for Python 2.4 compatibility) Blocks until all items in the Queue have been gotten and processed.
+
+        WARNING: Does nothing in Python 2.4 for now.
+        """
+        # TODO: Make this do something useful in Python 2.4
+        if hasattr(Queue, 'join'):
+            return Queue.join(self)
+
+    def wait(self):
+        "DEPRECATED: Use `join()` instead. Block until all jobs are completed."
+        self.join()
+
+GitUpdaterQueue = UpdaterQueue(0)
+
+
+# NB The following class and bits for running git commands were "liberated"
+# from git_multimail.py
+
+class CommandError(Exception):
+    def __init__(self, cmd, retcode):
+        self.cmd = cmd
+        self.retcode = retcode
+        Exception.__init__(
+            self,
+            'Command "%s" failed with retcode %s' % (' '.join(cmd), retcode,)
+            )
+
+# It is assumed in many places that the encoding is uniformly UTF-8,
+# so changing these constants is unsupported.  But define them here
+# anyway, to make it easier to find (at least most of) the places
+# where the encoding is important.
+(ENCODING, CHARSET) = ('UTF-8', 'utf-8')
+
+
+# The "git" program (this could be changed to include a full path):
+GIT_EXECUTABLE = 'git'
+
+
+# How "git" should be invoked (including global arguments), as a list
+# of words.  This variable is usually initialized automatically by
+# read_git_output() via choose_git_command(), but if a value is set
+# here then it will be used unconditionally.
+GIT_CMD = None
+
+
+def choose_git_command():
+    """Decide how to invoke git, and record the choice in GIT_CMD."""
+
+    global GIT_CMD
+
+    if GIT_CMD is None:
+        try:
+            # Check to see whether the "-c" option is accepted (it was
+            # only added in Git 1.7.2).  We don't actually use the
+            # output of "git --version", though if we needed more
+            # specific version information this would be the place to
+            # do it.
+            cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version']
+            read_output(cmd)
+            GIT_CMD = [GIT_EXECUTABLE, '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)]
+        except CommandError:
+            GIT_CMD = [GIT_EXECUTABLE]
+
+
+def read_git_output(args, input=None, keepends=False, **kw):
+    """Read the output of a Git command."""
+
+    if GIT_CMD is None:
+        choose_git_command()
+
+    return read_output(GIT_CMD + args, input=input, keepends=keepends, **kw)
+
+
+def read_output(cmd, input=None, keepends=False, **kw):
+    if input:
+        stdin = subprocess.PIPE
+    else:
+        stdin = None
+    p = subprocess.Popen(
+        cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw
+        )
+    (out, err) = p.communicate(input)
+    retcode = p.wait()
+    if retcode:
+        raise CommandError(cmd, retcode)
+    if not keepends:
+        out = out.rstrip('\n\r')
+    return out
+
+
+
+
+
+def run_git_command(args, **kw):
+    """Runs a git command, ignoring the output.
+    """
+
+    read_git_output(args, **kw)
+
+
+def run_command(args, **kw):
+    """Runs a git command, ignoring the output.
+    """
+
+    read_output(args, **kw)
+
+
+class GitUpdater(Thread):
+    def __init__(self, server, basedir, repoprefix, branch='master', cmd=''):
+        Thread.__init__(self)
+        self.server = server
+        self.basedir = basedir
+        self.repoprefix = repoprefix
+        self.branch = branch
+        self.cmd = cmd
+
+    def run(self):
+        while 42:
+            repo = GitUpdaterQueue.get()
+            if repo is None:
+                break
+            try:
+                print >> sys.stderr, "Got update request for '%s'" % repo
+                clonefolder = os.path.join(self.basedir, repo)
+                if self.repoprefix:
+                    if not repo.startswith(self.repoprefix):
+                       print >> sys.stderr, "Ignoring repo '%s' due to invalid prefix" % repo
+                       GitUpdaterQueue.task_done()
+                       continue
+                    clonefolder = os.path.join(self.basedir, repo[len(self.repoprefix):]) 
+                command = []
+                treeish = ''
+                changed = True
+                if not os.path.exists(clonefolder):
+                    cloneparent = os.path.dirname(clonefolder)
+                    if not os.path.exists(cloneparent):
+                        os.makedirs(cloneparent)
+                    cloneurl = self.server + '/' + repo
+                    command = ['clone']
+                    if '--mirror' == self.branch:
+                        command.append('--mirror')
+                    command.append(cloneurl)
+                    command.append(clonefolder)
+                    print >> sys.stderr, "Cloning repo '%s' ('%s' -> '%s')" % (repo, cloneurl, clonefolder)
+
+                    run_git_command(command)
+                    if not os.path.isdir(clonefolder):
+                        raise Exception("Clone folder '%s' is not a directory. Cloning failed or file in it's place?" % clonefolder)
+                    os.chdir(clonefolder)
+                    if '--mirror' != self.branch and 'master' != self.branch:
+                        command = ['checkout', '-t', 'origin/' + self.branch]
+                        run_git_command(command)
+                elif os.path.isdir(clonefolder):
+                    os.chdir(clonefolder)
+                    print >> sys.stderr, "Updating existing repo '%s' ('%s')" % (repo, clonefolder)
+                    command = ['remote', 'update']
+                    run_git_command(command)
+                    if '--mirror' != self.branch:
+                        sha1before = read_git_output(['rev-parse', 'refs/heads/' + self.branch])
+                        sha1after = read_git_output(['rev-parse', 'refs/remotes/origin/' + self.branch])
+                        if sha1before and sha1after:
+                            if sha1before == sha1after:
+                                changed = False
+                                print >> sys.stderr, "Repo '%s' update on branch '%s': No changed detected" % (repo, self.branch)
+                            else:
+                                treeish = sha1before + '..' + sha1after
+                                print >> sys.stderr, "Repo '%s' update on branch '%s': Treeish '%s'" % (repo, self.branch, treeish)
+                        else:
+                            print >> sys.stderr, "Repo '%s' update on branch '%s': Before or after sha1 could not be extracted." % (repo, self.branch)
+                        command = ['update-ref', 'refs/heads/' + self.branch, 'refs/remotes/origin/' + self.branch]
+                        run_git_command(command)
+                        command = ['checkout', '-f', self.branch]
+                        run_git_command(command)
+                else:
+                    raise Exception("Clone folder '%s' is appears to be a file :s" % clonefolder)
+
+                if changed and self.cmd:
+                    # Udate the info/web/last-modified file as used by cgit
+                    os.chdir(clonefolder)
+                    command = [self.cmd, repo]
+                    if treeish:
+                        command += [treeish]
+                    run_command(command)
+
+                print >> sys.stderr, "Update for '%s' complete." % repo
+            except Exception, e:
+                print >> sys.stderr, "Error processing repo '%s'" % repo
+                print >> sys.stderr, str(e)
+
+            GitUpdaterQueue.task_done()
+
+class TimeoutServer(BaseHTTPServer.HTTPServer):
+    def get_request(self):
+        result = self.socket.accept()
+        result[0].settimeout(10)
+        return result
+
+class PostHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_POST(self):
+        ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
+        repo = ""
+        try:
+          if ctype != 'x-git/repo':
+              self.send_response(415)
+              return
+
+          length = int(self.headers.getheader('content-length'))
+          if length < 1:
+              self.send_response(411)
+              return
+          if length > 1024:
+              self.send_response(413)
+              return
+          repo = self.rfile.read(length)
+
+          if re.match("^[-_/a-zA-Z0-9\+\.]+$", repo) is None:
+              self.send_response(400)
+              return
+
+          GitUpdaterQueue.put(repo)
+          self.send_response(202)
+        except:
+          print >> sys.stderr, "Error"
+
+
+def Demote(pidfile, uid, gid):
+    def result():
+        piddir = os.path.dirname(pidfile)
+        if not os.path.exists(piddir):
+            os.makedirs(piddir)
+        fd = open(pidfile, 'w')
+        fd.write(str(os.getpid()))
+        fd.close()
+
+        if uid and gid:
+            os.setgid(gid)
+            os.setuid(uid)
+    return result
+
+
+def daemonise(options, serverprefix, basefolder):
+    pw = None
+    uid = False
+    gid = False
+    if options.user:
+        pw = pwd.getpwnam(options.user)
+        uid = pw.pw_uid
+        gid = pw.pw_gid
+    else:
+        pw = pwd.getpwnam(os.getlogin())
+
+    user    = pw.pw_name
+    dirname = pw.pw_dir
+    env = {
+      'HOME': dirname,
+      'LOGNAME': user,
+      'PWD': dirname,
+      'USER': user,
+    }
+    if os.getenv('PATH') is not None:
+        env['PATH'] = os.getenv('PATH')
+    if os.getenv('PYTHONPATH') is not None:
+        env['PYTHONPATH'] = os.getenv('PYTHONPATH')
+
+    args = [os.path.abspath(sys.argv[0])]
+    args.append('-a')
+    args.append(options.addr)
+    args.append('-p')
+    args.append(str(options.port))
+    args.append('-r')
+    args.append(options.repoprefix)
+    args.append('-b')
+    args.append(options.branch)
+    args.append('-c')
+    args.append(options.cmd)
+    args.append(serverprefix)
+    args.append(basefolder)
+
+    subprocess.Popen(
+        args, preexec_fn=Demote(options.pidfile, uid, gid), cwd=dirname, env=env
+    )
+    exit(0)
+
+
+def main():
+    usage = "usage: %prog [options] <serverprefix> <basefolder>"
+    description = """Listen for repository names being posted via a simle HTTP interface and clone/update them.
+POST data simply via curl:
+e.g. curl --header 'Content-Type: x-git/repo' --data 'my/repo/name' http://localhost:8000
+"""
+    parser = OptionParser(usage=usage, description=description)
+    parser.add_option("-a", "--addr",
+                      type="string", dest="addr", default="0.0.0.0",
+                      help="The interface address to bind to")
+    parser.add_option("-p", "--port",
+                      type="int", dest="port", default=8000,
+                      help="The port to bind to")
+    parser.add_option("-r", "--repo-prefix",
+                      type="string", dest="repoprefix", default="",
+                      help="Only handle repositories with the following prefix. This SHOULD contain a trailing slash if it's a folder but SHOULD NOT include a leading slash")
+    parser.add_option("-b", "--branch",
+                      type="string", dest="branch", default="--mirror",
+                      help="The branch to track on clone. If you pass '--mirror' (the default) as the branch name we will clone as a bare mirror")
+    parser.add_option("-c", "--cmd",
+                      type="string", dest="cmd", default="",
+                      help="Third party command to exectue after updates. It will execute in the folder of the repo and if we're not in mirror mode, a treeish will be passed as the only argument containing the refs that changed otherwise the command will be run without any arguments")
+    parser.add_option("-d", "--pid-file",
+                      type="string", dest="pidfile", default="",
+                      help="Daemonise and write pidfile")
+    parser.add_option("-u", "--user",
+                      type="string", dest="user", default="",
+                      help="Drop privileges to the given user (must be run as root)")
+
+    (options, args) = parser.parse_args()
+    if len(args) < 2:
+        parser.error("Both the <serverprefix> and <basefolder> arguments must be supplied.")
+    if len(args) > 2:
+        parser.print_usage()
+        exit(1)
+
+    serverprefix = args[0]
+    basefolder = args[1]
+
+    if options.pidfile:
+        daemonise(options, serverprefix, basefolder)
+
+    if options.user:
+        parser.error("You can only specify a user if you're also deamonising (with a pid file).")
+
+    try:
+        print >> sys.stderr, "Server started"
+        srvr = TimeoutServer((options.addr, options.port), PostHandler)
+        GitUpdater(serverprefix, basefolder, options.repoprefix, options.branch, options.cmd).start()
+        srvr.serve_forever()
+    except KeyboardInterrupt:
+        GitUpdaterQueue.put(None)
+        srvr.socket.close()
+
+if __name__ == "__main__":
+    main()
diff --git a/modules/gitmirror/templates/on-the-pull.init b/modules/gitmirror/templates/on-the-pull.init
new file mode 100755
index 00000000..0c575b77
--- /dev/null
+++ b/modules/gitmirror/templates/on-the-pull.init
@@ -0,0 +1,60 @@
+#! /bin/bash
+#
+### BEGIN INIT INFO
+# Provides: on-the-pull
+# Required-Start: $network
+# Required-Stop: $network
+# Default-Start: 2 3 4 5
+# Short-Description: Keep git mirrors up-to-date via external triggers
+# Description: Keep git mirrors up-to-date via external triggers
+### END INIT INFO
+
+# Source function library.
+. /etc/init.d/functions
+
+pidfile=/var/run/on-the-pull/on-the-pull.pid
+prog=/usr/local/bin/on-the-pull
+args="--pid-file=$pidfile --user=git --cmd=/usr/local/bin/gitmirror-sync-metadata git://git.mageia.org/ /git"
+
+
+start() {
+	gprintf "Starting On-The-Pull Git Mirror Daemon: "
+	daemon --check on-the-pull --pidfile $pidfile $prog $args
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/on-the-pull
+	return $RETVAL
+}
+
+stop() {
+	gprintf "Stopping On-The-Pull Git Mirror Daemon: "
+	killproc -p $pidfile on-the-pull
+}
+
+restart() {
+	stop
+	start
+}
+
+case "$1" in
+	start)
+		start
+		;;
+	stop)
+		stop
+		;;
+	status)
+		status on-the-pull $pidfile
+		;;
+	restart|reload)
+		restart
+		;;
+	condrestart)
+			[ -f /var/lock/subsys/on-the-pull ] && restart || :
+			;;
+	*)
+		gprintf "Usage: %s {start|stop|status|restart|condrestart}\n" "$(basename $0)"
+		exit 1
+esac
+
+exit 0
diff --git a/modules/gitmirror/templates/rsync-metadata.sh b/modules/gitmirror/templates/rsync-metadata.sh
new file mode 100755
index 00000000..7176cc54
--- /dev/null
+++ b/modules/gitmirror/templates/rsync-metadata.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+REPO="$1"
+GITROOT="/git"
+RSYNCROOT="rsync://valstar.mageia.org/git"
+
+if [ ! -d "$GITROOT/$REPO" ]; then
+	echo "No repository found $REPO" >&2
+	exit 1
+fi
+
+cp -af "$GITROOT/$REPO/config" "$GITROOT/$REPO/config.orig"
+/usr/bin/rsync -a --delete --include="description" --include="config" --include="info" --include="info/web" --include="info/web/last-modified" --exclude="*" "$RSYNCROOT/$REPO/" "$GITROOT/$REPO/"
+cp -af "$GITROOT/$REPO/config" "$GITROOT/$REPO/config.upstream"
+mv -f "$GITROOT/$REPO/config.orig" "$GITROOT/$REPO/config"
+
+OWNER=$(git config --file "$GITROOT/$REPO/config.upstream" gitweb.owner)
+DESC=$(git config --file "$GITROOT/$REPO/config.upstream" gitweb.description)
+rm -f "$GITROOT/$REPO/config.upstream"
+
+CUROWNER=$(git config --file "$GITROOT/$REPO/config" gitweb.owner)
+if [ "$CUROWNER" != "$OWNER" ]; then
+	git config --file "$GITROOT/$REPO/config" gitweb.owner "$OWNER"
+fi
+
+CURDESC=$(git config --file "$GITROOT/$REPO/config" gitweb.description)
+if [ "$CURDESC" != "$DESC" ]; then
+	git config --file "$GITROOT/$REPO/config" gitweb.owner "$DESC"
+fi