make --incremental robust to inconsistent import directories

An interrupted import (say with Ctrl-C) can leave the import directory in an
inconsistent state.  This can be due to checkpointing fast-import only
occassionally, but updating log-* files immediately, and/or other reasons.

The incremental mode can detect certain such situations and rewind back to a
safe state.  Note that since the default commit-interval is quite large, this
rewind can end up backtracking a lot.

Note also that import interrupted under the control of svn2git, say, for
missing rules should leave the import directory in a consistent state for
the purpose of svn2git.

3 files changed, 94 insertions, 15 deletions

diff --git a/src/main.cpp b/src/main.cpp
index da39a5b..5d0a1fc 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -19,7 +19,9 @@
 #include <QFile>
 #include <QStringList>
 #include <QTextStream>
+#include <QDebug>
 
+#include <limits.h>
 #include <stdio.h>
 
 #include "CommandLineParser.h"
@@ -119,18 +121,34 @@ int main(int argc, char **argv)
     QHash<QString, Repository *> repositories;
     bool incremental = args->contains("incremental");
 
-    int min_rev = resume_from;
+    int cutoff = resume_from ? resume_from : INT_MAX;
+ retry:
+    int min_rev = 0;
     foreach (Rules::Repository rule, rules.repositories()) {
         Repository *repo = new Repository(rule);
         repositories.insert(rule.name, repo);
 
 	if (incremental) {
-	    int repo_next = repo->setupIncremental(resume_from);
+	    int repo_next = repo->setupIncremental(cutoff);
+	    if (cutoff < min_rev) {
+		qWarning() << "rewinding; did you hit Ctrl-C?";
+		goto retry;
+	    }
 	    if (min_rev < repo_next)
 		min_rev = repo_next;
 	}
     }
 
+    if (incremental && resume_from) {
+	if (cutoff < resume_from) {
+	    qCritical() << "Cannot resume from" << resume_from << "as there are errors in revision" << cutoff;
+	    return EXIT_FAILURE;
+	}
+	if (min_rev < resume_from)
+	    qDebug() << "skipping revisions" << min_rev << "to" << resume_from - 1 << "as requested";
+	min_rev = resume_from;
+    }
+
     if (min_rev < 1)
         min_rev = 1;
 
diff --git a/src/repository.cpp b/src/repository.cpp
index 18deab5..f57cdbb 100644
--- a/src/repository.cpp
+++ b/src/repository.cpp
@@ -92,7 +92,55 @@ static QString logFileName(QString name)
     return name;
 }
 
-int Repository::setupIncremental(int resume_from)
+static int lastValidMark(QString name)
+{
+    QFile marksfile(name + "/info/fast-import/marks");
+    if (!marksfile.open(QIODevice::ReadOnly))
+	return 0;
+
+    int prev_mark = 0;
+
+    int lineno = 0;
+    while (!marksfile.atEnd()) {
+	QString line = marksfile.readLine();
+	++lineno;
+	if (line.isEmpty())
+	    continue;
+
+	int mark = 0;
+	if (line[0] == ':') {
+	    int sp = line.indexOf(' ');
+	    if (sp != -1) {
+		QString m = line.mid(1, sp-1);
+		mark = m.toInt();
+	    }
+	}
+
+	if (!mark) {
+	    qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?";
+	    return 0;
+	}
+
+	if (mark == prev_mark) {
+	    qCritical() << marksfile.fileName() << "line" << lineno << "marks file has duplicates";
+	    return 0;
+	}
+
+	if (mark < prev_mark) {
+	    qCritical() << marksfile.fileName() << "line" << lineno << "marks file not sorted";
+	    return 0;
+	}
+
+	if (mark > prev_mark + 1)
+	    break;
+
+	prev_mark = mark;
+    }
+
+    return prev_mark;
+}
+
+int Repository::setupIncremental(int &cutoff)
 {
     QFile logfile(logFileName(name));
     if (!logfile.exists())
@@ -102,10 +150,13 @@ int Repository::setupIncremental(int resume_from)
 
     QRegExp progress("progress SVN r(\\d+) branch (.*) = :(\\d+)");
 
+    int last_valid_mark = lastValidMark(name);
+
     int last_revnum = 0;
+    qint64 pos = 0;
 
     while (!logfile.atEnd()) {
-        qint64 pos = logfile.pos();
+        pos = logfile.pos();
         QByteArray line = logfile.readLine();
         int hash = line.indexOf('#');
         if (hash != -1)
@@ -120,22 +171,21 @@ int Repository::setupIncremental(int resume_from)
         QString branch = progress.cap(2);
         int mark = progress.cap(3).toInt();
 
-        if (resume_from && revnum >= resume_from) {
-            // backup file, since we'll truncate
-            QString bkup = logfile.fileName() + ".old";
-            QFile::remove(bkup);
-            logfile.copy(bkup);
-
-            // truncate, so that we ignore the rest of the revisions
-            logfile.resize(pos);
-            return resume_from;
-        }
+        if (revnum >= cutoff)
+	    goto beyond_cutoff;
 
         if (revnum < last_revnum)
             qWarning() << name << "revision numbers are not monotonic: "
                        << "got" << QString::number(last_revnum)
                        << "and then" << QString::number(revnum);
 
+
+	if (mark > last_valid_mark) {
+	    qWarning() << name << "unknown commit mark found: rewinding";
+	    cutoff = revnum;
+	    goto beyond_cutoff;
+	}
+
         last_revnum = revnum;
 
 	if (last_commit_mark < mark)
@@ -149,6 +199,17 @@ int Repository::setupIncremental(int resume_from)
     }
 
     return last_revnum + 1;
+
+  beyond_cutoff:
+    // backup file, since we'll truncate
+    QString bkup = logfile.fileName() + ".old";
+    QFile::remove(bkup);
+    logfile.copy(bkup);
+
+    // truncate, so that we ignore the rest of the revisions
+    qDebug() << name << "truncating history to" << cutoff;
+    logfile.resize(pos);
+    return cutoff;
 }
 
 Repository::~Repository()
diff --git a/src/repository.h b/src/repository.h
index 728b2ab..9c2447f 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -60,7 +60,7 @@ public:
         QIODevice *addFile(const QString &path, int mode, qint64 length);
     };
     Repository(const Rules::Repository &rule);
-    int setupIncremental(int resume_from);
+    int setupIncremental(int &cutoff);
     ~Repository();
 
     void reloadBranches();