Reduce size of fast-import marks file by not persisting file-level marks

Use two allocators for marks, one persistent commit counter that starts at 0
and counts up commits, and a transitional counter, for files, that counts down
from maxMark, and is reset on each SVN revision.

Note that the marks file will still have marks for some, but not all, files.
The number of such marks is limited by the size of the SVN revision that affects
the most files.

For instance, this changed the size of one marks file from 19M to 3.2M.

fast-import issues: We currently set maxMark = (1<<20)-1.  Anything large seems
to trigger a bug in the sparse array dumping routine in git-fast-import in
certain versions of git.

2 files changed, 33 insertions, 7 deletions

diff --git a/src/repository.cpp b/src/repository.cpp
index bd2931a..18deab5 100644
--- a/src/repository.cpp
+++ b/src/repository.cpp
@@ -25,6 +25,8 @@
 
 static const int maxSimultaneousProcesses = 100;
 
+static const int maxMark = (1 << 20) - 1; // some versions of git-fast-import are buggy for larger values of maxMark
+
 class ProcessCache: QLinkedList<Repository *>
 {
 public:
@@ -52,7 +54,7 @@ public:
 static ProcessCache processCache;
 
 Repository::Repository(const Rules::Repository &rule)
-  : name(rule.name), commitCount(0), outstandingTransactions(0),  lastmark(0), processHasStarted(false)
+    : name(rule.name), commitCount(0), outstandingTransactions(0),  last_commit_mark(0), next_file_mark(maxMark), processHasStarted(false)
 {
     foreach (Rules::Repository::Branch branchRule, rule.branches) {
         Branch branch;
@@ -136,8 +138,8 @@ int Repository::setupIncremental(int resume_from)
 
         last_revnum = revnum;
 
-	if (lastmark < mark)
-	    lastmark = mark;
+	if (last_commit_mark < mark)
+	    last_commit_mark = mark;
 
         Branch &br = branches[branch];
         if (!br.created || !mark || !br.marks.last())
@@ -303,6 +305,12 @@ Repository::Transaction *Repository::newTransaction(const QString &branch, const
     return txn;
 }
 
+void Repository::forgetTransaction(Transaction *)
+{
+    if (!--outstandingTransactions)
+	next_file_mark = maxMark;
+}
+
 void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix,
                                     int revnum,
                                     const QByteArray &author, uint dt,
@@ -396,7 +404,7 @@ void Repository::startFastImport()
 
 Repository::Transaction::~Transaction()
 {
-    --repository->outstandingTransactions;
+    repository->forgetTransaction(this);
 }
 
 void Repository::Transaction::setAuthor(const QByteArray &a)
@@ -457,7 +465,10 @@ void Repository::Transaction::deleteFile(const QString &path)
 
 QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length)
 {
-    int mark = ++repository->lastmark;
+    int mark = repository->next_file_mark--;
+
+    // in case the two mark allocations meet, we might as well just abort
+    Q_ASSERT(mark > repository->last_commit_mark + 1);
 
     if (modifiedFiles.capacity() == 0)
         modifiedFiles.reserve(2048);
@@ -484,7 +495,13 @@ void Repository::Transaction::commit()
 {
     processCache.touch(repository);
 
-    int mark = ++repository->lastmark;
+    // We might be tempted to use the SVN revision number as the fast-import commit mark.
+    // However, a single SVN revision can modify multple branches, and thus lead to multiple
+    // commits in the same repo.  So, we need to maintain a separate commit mark counter.
+    int mark = ++repository->last_commit_mark;
+
+    // in case the two mark allocations meet, we might as well just abort
+    Q_ASSERT(mark < repository->next_file_mark - 1);
 
     // create the commit message
     QByteArray message = log;
diff --git a/src/repository.h b/src/repository.h
index f7beb99..728b2ab 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -97,12 +97,21 @@ private:
     QProcess fastImport;
     int commitCount;
     int outstandingTransactions;
-    int lastmark;
+
+    /* starts at 0, and counts up.  */
+    int last_commit_mark;
+
+    /* starts at maxMark and counts down. Reset after each SVN revision */
+    int next_file_mark;
+
     bool processHasStarted;
 
     void startFastImport();
     void closeFastImport();
 
+    // called when a transaction is deleted
+    void forgetTransaction(Transaction *t);
+
     friend class ProcessCache;
     Q_DISABLE_COPY(Repository)
 };