Introduce incremental mode with --incremental flag

We use the progress logs that we carefully maintained to recreate the
branch data structures, as described in earlier commits.

A major change/improvement is that reloadBranches() now uses the marks file
and internal data structures to prime the fast-import rather than using
git-rev-parse.

We also handle --resume-from properly, by truncating the log file to revisions
that only precede the revision resumed from.  Note that git fast-import allows
marks to be reused without any extra processing.

3 files changed, 92 insertions, 21 deletions

diff --git a/src/main.cpp b/src/main.cpp
index c328ef1..da39a5b 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -70,6 +70,7 @@ static const CommandLineOption options[] = {
     {"--dry-run", "don't actually write anything"},
     {"--debug-rules", "print what rule is being used for each file"},
     {"--commit-interval NUMBER", "if passed the cache will be flushed to git every NUMBER of commits"},
+    {"--incremental", "attempts to restart an import based on information in log-* and git-fast-import"},
     {"-h, --help", "show help"},
     {"-v, --version", "show version"},
     CommandLineLastOption
@@ -111,18 +112,28 @@ int main(int argc, char **argv)
     Rules rules(args->optionArgument(QLatin1String("rules")));
     rules.load();
 
-    int min_rev = args->optionArgument(QLatin1String("resume-from")).toInt();
+    int resume_from = args->optionArgument(QLatin1String("resume-from")).toInt();
     int max_rev = args->optionArgument(QLatin1String("max-rev")).toInt();
-    if (min_rev < 1)
-        min_rev = 1;
 
     // create the repository list
     QHash<QString, Repository *> repositories;
+    bool incremental = args->contains("incremental");
+
+    int min_rev = resume_from;
     foreach (Rules::Repository rule, rules.repositories()) {
         Repository *repo = new Repository(rule);
         repositories.insert(rule.name, repo);
+
+	if (incremental) {
+	    int repo_next = repo->setupIncremental(resume_from);
+	    if (min_rev < repo_next)
+		min_rev = repo_next;
+	}
     }
 
+    if (min_rev < 1)
+        min_rev = 1;
+
     Svn::initialize();
     Svn svn(args->arguments().first());
     svn.setMatchRules(rules.matchRules());
diff --git a/src/repository.cpp b/src/repository.cpp
index 847771f..cc96e7b 100644
--- a/src/repository.cpp
+++ b/src/repository.cpp
@@ -83,6 +83,72 @@ Repository::Repository(const Rules::Repository &rule)
     }
 }
 
+static QString logFileName(QString name)
+{
+    name.replace('/', '_');
+    name.prepend("log-");
+    return name;
+}
+
+int Repository::setupIncremental(int resume_from)
+{
+    QFile logfile(logFileName(name));
+    if (!logfile.exists())
+        return 1;
+
+    logfile.open(QIODevice::ReadWrite);
+
+    QRegExp progress("progress SVN r(\\d+) branch (.*) = :(\\d+)");
+
+    int last_revnum = 0;
+
+    while (!logfile.atEnd()) {
+        qint64 pos = logfile.pos();
+        QByteArray line = logfile.readLine();
+        int hash = line.indexOf('#');
+        if (hash != -1)
+            line.truncate(hash);
+        line = line.trimmed();
+        if (line.isEmpty())
+            continue;
+        if (!progress.exactMatch(line))
+            continue;
+
+        int revnum = progress.cap(1).toInt();
+        QString branch = progress.cap(2);
+        int mark = progress.cap(3).toInt();
+
+        if (resume_from && revnum >= resume_from) {
+            // backup file, since we'll truncate
+            QString bkup = logfile.fileName() + ".old";
+            QFile::remove(bkup);
+            logfile.copy(bkup);
+
+            // truncate, so that we ignore the rest of the revisions
+            logfile.resize(pos);
+            return resume_from;
+        }
+
+        if (revnum < last_revnum)
+            qWarning() << name << "revision numbers are not monotonic: "
+                       << "got" << QString::number(last_revnum)
+                       << "and then" << QString::number(revnum);
+
+        last_revnum = revnum;
+
+	if (lastmark < mark)
+	    lastmark = mark;
+
+        Branch &br = branches[branch];
+        if (!br.created || !mark || !br.marks.last())
+            br.created = revnum;
+        br.commits.append(revnum);
+        br.marks.append(mark);
+    }
+
+    return last_revnum + 1;
+}
+
 Repository::~Repository()
 {
     Q_ASSERT(outstandingTransactions == 0);
@@ -107,23 +173,19 @@ void Repository::closeFastImport()
 
 void Repository::reloadBranches()
 {
-    QProcess revParse;
-    revParse.setWorkingDirectory(name);
-    revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
-    revParse.waitForFinished(-1);
-
-    if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
-        while (revParse.canReadLine()) {
-            QByteArray branchName = revParse.readLine().trimmed();
+    foreach (QString branch, branches.keys()) {
+        Branch &br = branches[branch];
 
-            //qDebug() << "Repo" << name << "reloaded branch" << branchName;
+        if (!br.marks.count() || !br.marks.last())
+	    continue;
 
-	    Q_ASSERT(branches[branchName].created);
+        QByteArray branchRef = branch.toUtf8();
+        if (!branchRef.startsWith("refs/"))
+            branchRef.prepend("refs/heads/");
 
-            fastImport.write("reset refs/heads/" + branchName +
-                             "\nfrom refs/heads/" + branchName + "^0\n\n"
-                             "progress Branch refs/heads/" + branchName + " reloaded\n");
-        }
+	fastImport.write("reset " + branchRef +
+			 "\nfrom :" + QByteArray::number(br.marks.last()) + "\n\n"
+			 "progress Branch " + branchRef + " reloaded\n");
     }
 }
 
@@ -318,10 +380,7 @@ void Repository::startFastImport()
         processHasStarted = true;
 
         // start the process
-        QString outputFile = name;
-        outputFile.replace('/', '_');
-        outputFile.prepend("log-");
-        fastImport.setStandardOutputFile(outputFile, QIODevice::Append);
+        fastImport.setStandardOutputFile(logFileName(name), QIODevice::Append);
         fastImport.setProcessChannelMode(QProcess::MergedChannels);
 
         if (!CommandLineParser::instance()->contains("dry-run")) {
diff --git a/src/repository.h b/src/repository.h
index 65802bb..f7beb99 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -60,6 +60,7 @@ public:
         QIODevice *addFile(const QString &path, int mode, qint64 length);
     };
     Repository(const Rules::Repository &rule);
+    int setupIncremental(int resume_from);
     ~Repository();
 
     void reloadBranches();