aboutsummaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/main.cpp63
-rw-r--r--src/repository.cpp621
-rw-r--r--src/repository.h92
-rw-r--r--src/ruleparser.cpp13
-rw-r--r--src/ruleparser.h3
-rw-r--r--src/svn.cpp162
6 files changed, 702 insertions, 252 deletions
diff --git a/src/main.cpp b/src/main.cpp
index 0edf8f5..f2b189e 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -19,7 +19,9 @@
#include <QFile>
#include <QStringList>
#include <QTextStream>
+#include <QDebug>
+#include <limits.h>
#include <stdio.h>
#include "CommandLineParser.h"
@@ -115,18 +117,59 @@ int main(int argc, char **argv)
Rules rules(args->optionArgument(QLatin1String("rules")));
rules.load();
- int min_rev = args->optionArgument(QLatin1String("resume-from")).toInt();
+ int resume_from = args->optionArgument(QLatin1String("resume-from")).toInt();
int max_rev = args->optionArgument(QLatin1String("max-rev")).toInt();
- if (min_rev < 1)
- min_rev = 1;
// create the repository list
QHash<QString, Repository *> repositories;
+
+ int cutoff = resume_from ? resume_from : INT_MAX;
+ retry:
+ int min_rev = 1;
foreach (Rules::Repository rule, rules.repositories()) {
- Repository *repo = new Repository(rule);
+ Repository *repo = makeRepository(rule, repositories);
+ if (!repo)
+ return EXIT_FAILURE;
repositories.insert(rule.name, repo);
+
+ int repo_next = repo->setupIncremental(cutoff);
+
+ /*
+ * cutoff < resume_from => error exit eventually
+ * repo_next == cutoff => probably truncated log
+ */
+ if (cutoff < resume_from && repo_next == cutoff)
+ /*
+ * Restore the log file so we fail the next time
+ * svn2git is invoked with the same arguments
+ */
+ repo->restoreLog();
+
+ if (cutoff < min_rev)
+ /*
+ * We've rewound before the last revision of some
+ * repository that we've already seen. Start over
+ * from the beginning. (since cutoff is decreasing,
+ * we're sure we'll make forward progress eventually)
+ */
+ goto retry;
+
+ if (min_rev < repo_next)
+ min_rev = repo_next;
+ }
+
+ if (cutoff < resume_from) {
+ qCritical() << "Cannot resume from" << resume_from
+ << "as there are errors in revision" << cutoff;
+ return EXIT_FAILURE;
}
+ if (min_rev < resume_from)
+ qDebug() << "skipping revisions" << min_rev << "to" << resume_from - 1 << "as requested";
+
+ if (resume_from)
+ min_rev = resume_from;
+
Svn::initialize();
Svn svn(args->arguments().first());
svn.setMatchRules(rules.matchRules());
@@ -135,15 +178,19 @@ int main(int argc, char **argv)
if (max_rev < 1)
max_rev = svn.youngestRevision();
- for (int i = min_rev; i <= max_rev; ++i)
- if (!svn.exportRevision(i))
+
+ bool errors = false;
+ for (int i = min_rev; i <= max_rev; ++i) {
+ if (!svn.exportRevision(i)) {
+ errors = true;
break;
+ }
+ }
foreach (Repository *repo, repositories) {
repo->finalizeTags();
delete repo;
}
- // success
- return 0;
+ return errors ? EXIT_FAILURE : EXIT_SUCCESS;
}
diff --git a/src/repository.cpp b/src/repository.cpp
index be73df6..5bcd8dd 100644
--- a/src/repository.cpp
+++ b/src/repository.cpp
@@ -25,10 +25,161 @@
static const int maxSimultaneousProcesses = 100;
-class ProcessCache: QLinkedList<Repository *>
+static const int maxMark = (1 << 20) - 1; // some versions of git-fast-import are buggy for larger values of maxMark
+
+class FastImportRepository : public Repository
+{
+public:
+ class Transaction : public Repository::Transaction
+ {
+ Q_DISABLE_COPY(Transaction)
+ friend class FastImportRepository;
+
+ FastImportRepository *repository;
+ QByteArray branch;
+ QByteArray svnprefix;
+ QByteArray author;
+ QByteArray log;
+ uint datetime;
+ int revnum;
+
+ QVector<int> merges;
+
+ QStringList deletedFiles;
+ QByteArray modifiedFiles;
+
+ inline Transaction() {}
+ public:
+ ~Transaction();
+ void commit();
+
+ void setAuthor(const QByteArray &author);
+ void setDateTime(uint dt);
+ void setLog(const QByteArray &log);
+
+ void noteCopyFromBranch (const QString &prevbranch, int revFrom);
+
+ void deleteFile(const QString &path);
+ QIODevice *addFile(const QString &path, int mode, qint64 length);
+ };
+ FastImportRepository(const Rules::Repository &rule);
+ int setupIncremental(int &cutoff);
+ void restoreLog();
+ ~FastImportRepository();
+
+ void reloadBranches();
+ int createBranch(const QString &branch, int revnum,
+ const QString &branchFrom, int revFrom);
+ int deleteBranch(const QString &branch, int revnum);
+ Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum);
+
+ void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum,
+ const QByteArray &author, uint dt,
+ const QByteArray &log);
+ void finalizeTags();
+
+private:
+ struct Branch
+ {
+ int created;
+ QVector<int> commits;
+ QVector<int> marks;
+ };
+ struct AnnotatedTag
+ {
+ QString supportingRef;
+ QByteArray svnprefix;
+ QByteArray author;
+ QByteArray log;
+ uint dt;
+ int revnum;
+ };
+
+ QHash<QString, Branch> branches;
+ QHash<QString, AnnotatedTag> annotatedTags;
+ QString name;
+ QProcess fastImport;
+ int commitCount;
+ int outstandingTransactions;
+
+ /* starts at 0, and counts up. */
+ int last_commit_mark;
+
+ /* starts at maxMark and counts down. Reset after each SVN revision */
+ int next_file_mark;
+
+ bool processHasStarted;
+
+ void startFastImport();
+ void closeFastImport();
+
+ // called when a transaction is deleted
+ void forgetTransaction(Transaction *t);
+
+ int resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment);
+ int markFrom(const QString &branchFrom, int branchRevNum, QByteArray &desc);
+
+ friend class ProcessCache;
+ Q_DISABLE_COPY(FastImportRepository)
+};
+
+class PrefixingRepository : public Repository
+{
+ Repository *repo;
+ QString prefix;
+public:
+ class Transaction : public Repository::Transaction
+ {
+ Q_DISABLE_COPY(Transaction)
+
+ Repository::Transaction *txn;
+ QString prefix;
+ public:
+ Transaction(Repository::Transaction *t, const QString &p) : txn(t), prefix(p) {}
+ ~Transaction() { delete txn; }
+ void commit() { txn->commit(); }
+
+ void setAuthor(const QByteArray &author) { txn->setAuthor(author); }
+ void setDateTime(uint dt) { txn->setDateTime(dt); }
+ void setLog(const QByteArray &log) { txn->setLog(log); }
+
+ void noteCopyFromBranch (const QString &prevbranch, int revFrom)
+ { txn->noteCopyFromBranch(prevbranch, revFrom); }
+
+ void deleteFile(const QString &path) { txn->deleteFile(prefix + path); }
+ QIODevice *addFile(const QString &path, int mode, qint64 length)
+ { return txn->addFile(prefix + path, mode, length); }
+ };
+
+ PrefixingRepository(Repository *r, const QString &p) : repo(r), prefix(p) {}
+
+ int setupIncremental(int &) { return 1; }
+ void restoreLog() {}
+
+ int createBranch(const QString &branch, int revnum,
+ const QString &branchFrom, int revFrom)
+ { return repo->createBranch(branch, revnum, branchFrom, revFrom); }
+
+ int deleteBranch(const QString &branch, int revnum)
+ { return repo->deleteBranch(branch, revnum); }
+
+ Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum)
+ {
+ Repository::Transaction *t = repo->newTransaction(branch, svnprefix, revnum);
+ return new Transaction(t, prefix);
+ }
+
+ void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum,
+ const QByteArray &author, uint dt,
+ const QByteArray &log)
+ { repo->createAnnotatedTag(name, svnprefix, revnum, author, dt, log); }
+ void finalizeTags() { /* loop that called this will invoke it on 'repo' too */ }
+};
+
+class ProcessCache: QLinkedList<FastImportRepository *>
{
public:
- void touch(Repository *repo)
+ void touch(FastImportRepository *repo)
{
remove(repo);
@@ -40,7 +191,7 @@ public:
append(repo);
}
- inline void remove(Repository *repo)
+ inline void remove(FastImportRepository *repo)
{
#if QT_VERSION >= 0x040400
removeOne(repo);
@@ -51,8 +202,27 @@ public:
};
static ProcessCache processCache;
-Repository::Repository(const Rules::Repository &rule)
- : name(rule.name), commitCount(0), outstandingTransactions(0), lastmark(0), processHasStarted(false)
+Repository *makeRepository(const Rules::Repository &rule, const QHash<QString, Repository *> &repositories)
+{
+ if (rule.forwardTo.isEmpty())
+ return new FastImportRepository(rule);
+ Repository *r = repositories[rule.forwardTo];
+ if (!r) {
+ qCritical() << "no repository with name" << rule.forwardTo << "found at line" << rule.lineNumber;
+ return r;
+ }
+ return new PrefixingRepository(r, rule.prefix);
+}
+
+static QString marksFileName(QString name)
+{
+ name.replace('/', '_');
+ name.prepend("marks-");
+ return name;
+}
+
+FastImportRepository::FastImportRepository(const Rules::Repository &rule)
+ : name(rule.name), commitCount(0), outstandingTransactions(0), last_commit_mark(0), next_file_mark(maxMark), processHasStarted(false)
{
foreach (Rules::Repository::Branch branchRule, rule.branches) {
Branch branch;
@@ -73,17 +243,167 @@ Repository::Repository(const Rules::Repository &rule)
init.setWorkingDirectory(name);
init.start("git", QStringList() << "--bare" << "init");
init.waitForFinished(-1);
+ {
+ QFile marks(name + "/" + marksFileName(name));
+ marks.open(QIODevice::WriteOnly);
+ marks.close();
+ }
}
}
}
-Repository::~Repository()
+static QString logFileName(QString name)
+{
+ name.replace('/', '_');
+ name.prepend("log-");
+ return name;
+}
+
+static int lastValidMark(QString name)
+{
+ QFile marksfile(name + "/" + marksFileName(name));
+ if (!marksfile.open(QIODevice::ReadOnly))
+ return 0;
+
+ int prev_mark = 0;
+
+ int lineno = 0;
+ while (!marksfile.atEnd()) {
+ QString line = marksfile.readLine();
+ ++lineno;
+ if (line.isEmpty())
+ continue;
+
+ int mark = 0;
+ if (line[0] == ':') {
+ int sp = line.indexOf(' ');
+ if (sp != -1) {
+ QString m = line.mid(1, sp-1);
+ mark = m.toInt();
+ }
+ }
+
+ if (!mark) {
+ qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?";
+ return 0;
+ }
+
+ if (mark == prev_mark) {
+ qCritical() << marksfile.fileName() << "line" << lineno << "marks file has duplicates";
+ return 0;
+ }
+
+ if (mark < prev_mark) {
+ qCritical() << marksfile.fileName() << "line" << lineno << "marks file not sorted";
+ return 0;
+ }
+
+ if (mark > prev_mark + 1)
+ break;
+
+ prev_mark = mark;
+ }
+
+ return prev_mark;
+}
+
+int FastImportRepository::setupIncremental(int &cutoff)
+{
+ QFile logfile(logFileName(name));
+ if (!logfile.exists())
+ return 1;
+
+ logfile.open(QIODevice::ReadWrite);
+
+ QRegExp progress("progress SVN r(\\d+) branch (.*) = :(\\d+)");
+
+ int last_valid_mark = lastValidMark(name);
+
+ int last_revnum = 0;
+ qint64 pos = 0;
+ int retval = 0;
+ QString bkup = logfile.fileName() + ".old";
+
+ while (!logfile.atEnd()) {
+ pos = logfile.pos();
+ QByteArray line = logfile.readLine();
+ int hash = line.indexOf('#');
+ if (hash != -1)
+ line.truncate(hash);
+ line = line.trimmed();
+ if (line.isEmpty())
+ continue;
+ if (!progress.exactMatch(line))
+ continue;
+
+ int revnum = progress.cap(1).toInt();
+ QString branch = progress.cap(2);
+ int mark = progress.cap(3).toInt();
+
+ if (revnum >= cutoff)
+ goto beyond_cutoff;
+
+ if (revnum < last_revnum)
+ qWarning() << name << "revision numbers are not monotonic: "
+ << "got" << QString::number(last_revnum)
+ << "and then" << QString::number(revnum);
+
+ if (mark > last_valid_mark) {
+ qWarning() << name << "unknown commit mark found: rewinding -- did you hit Ctrl-C?";
+ cutoff = revnum;
+ goto beyond_cutoff;
+ }
+
+ last_revnum = revnum;
+
+ if (last_commit_mark < mark)
+ last_commit_mark = mark;
+
+ Branch &br = branches[branch];
+ if (!br.created || !mark || !br.marks.last())
+ br.created = revnum;
+ br.commits.append(revnum);
+ br.marks.append(mark);
+ }
+
+ retval = last_revnum + 1;
+ if (retval == cutoff)
+ /*
+ * If a stale backup file exists already, remove it, so that
+ * we don't confuse ourselves in 'restoreLog()'
+ */
+ QFile::remove(bkup);
+
+ return retval;
+
+ beyond_cutoff:
+ // backup file, since we'll truncate
+ QFile::remove(bkup);
+ logfile.copy(bkup);
+
+ // truncate, so that we ignore the rest of the revisions
+ qDebug() << name << "truncating history to revision" << cutoff;
+ logfile.resize(pos);
+ return cutoff;
+}
+
+void FastImportRepository::restoreLog()
+{
+ QString file = logFileName(name);
+ QString bkup = file + ".old";
+ if (!QFile::exists(bkup))
+ return;
+ QFile::remove(file);
+ QFile::rename(bkup, file);
+}
+
+FastImportRepository::~FastImportRepository()
{
Q_ASSERT(outstandingTransactions == 0);
closeFastImport();
}
-void Repository::closeFastImport()
+void FastImportRepository::closeFastImport()
{
if (fastImport.state() != QProcess::NotRunning) {
fastImport.write("checkpoint\n");
@@ -97,43 +417,52 @@ void Repository::closeFastImport()
}
processHasStarted = false;
processCache.remove(this);
- // Save the exported marks
- QString revsFile = name;
- revsFile.replace('/', '_');
- revsFile.prepend("revisions-");
- QFile exportedMarks(revsFile);
- qDebug() << exportedMarks.open(QIODevice::Truncate | QIODevice::Text | QIODevice::WriteOnly);
-
- int mark;
- foreach(mark, commitMarks.keys())
- {
- exportedMarks.write(QString(":%2 r%1\n").arg(mark).arg(commitMarks.value(mark)).toLocal8Bit());
+}
+
+void FastImportRepository::reloadBranches()
+{
+ foreach (QString branch, branches.keys()) {
+ Branch &br = branches[branch];
+
+ if (!br.marks.count() || !br.marks.last())
+ continue;
+
+ QByteArray branchRef = branch.toUtf8();
+ if (!branchRef.startsWith("refs/"))
+ branchRef.prepend("refs/heads/");
+
+ fastImport.write("reset " + branchRef +
+ "\nfrom :" + QByteArray::number(br.marks.last()) + "\n\n"
+ "progress Branch " + branchRef + " reloaded\n");
}
- exportedMarks.close();
}
-void Repository::reloadBranches()
+int FastImportRepository::markFrom(const QString &branchFrom, int branchRevNum, QByteArray &branchFromDesc)
{
- QProcess revParse;
- revParse.setWorkingDirectory(name);
- revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
- revParse.waitForFinished(-1);
-
- if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
- while (revParse.canReadLine()) {
- QByteArray branchName = revParse.readLine().trimmed();
-
- //qDebug() << "Repo" << name << "reloaded branch" << branchName;
- branches[branchName].created = 1;
- fastImport.write("reset refs/heads/" + branchName +
- "\nfrom refs/heads/" + branchName + "^0\n\n"
- "progress Branch refs/heads/" + branchName + " reloaded\n");
- }
+ Branch &brFrom = branches[branchFrom];
+ if (!brFrom.created)
+ return -1;
+
+ if (branchRevNum == brFrom.commits.last())
+ return brFrom.marks.last();
+
+ QVector<int>::const_iterator it = qUpperBound(brFrom.commits, branchRevNum);
+ if (it == brFrom.commits.begin())
+ return 0;
+
+ int closestCommit = *--it;
+
+ if (!branchFromDesc.isEmpty()) {
+ branchFromDesc += " at r" + QByteArray::number(branchRevNum);
+ if (closestCommit != branchRevNum)
+ branchFromDesc += " => r" + QByteArray::number(closestCommit);
}
+
+ return brFrom.marks[it - brFrom.commits.begin()];
}
-void Repository::createBranch(const QString &branch, int revnum,
- const QString &branchFrom, int branchRevNum)
+int FastImportRepository::createBranch(const QString &branch, int revnum,
+ const QString &branchFrom, int branchRevNum)
{
startFastImport();
if (!branches.contains(branch)) {
@@ -141,64 +470,64 @@ void Repository::createBranch(const QString &branch, int revnum,
<< "Going to create it automatically";
}
- QByteArray branchRef = branch.toUtf8();
- if (!branchRef.startsWith("refs/"))
- branchRef.prepend("refs/heads/");
-
-
- Branch &br = branches[branch];
- if (br.created && br.created != revnum) {
- QByteArray backupBranch = branchRef + '_' + QByteArray::number(revnum);
- qWarning() << branch << "already exists; backing up to" << backupBranch;
+ QByteArray branchFromDesc = "from branch " + branchFrom.toUtf8();
+ int mark = markFrom(branchFrom, branchRevNum, branchFromDesc);
- fastImport.write("reset " + backupBranch + "\nfrom " + branchRef + "\n\n");
+ if (mark == -1) {
+ qCritical() << branch << "in repository" << name
+ << "is branching from branch" << branchFrom
+ << "but the latter doesn't exist. Can't continue.";
+ return EXIT_FAILURE;
}
- // now create the branch
- br.created = revnum;
- QByteArray branchFromRef;
- const int closestCommit = *qLowerBound(commitMarks.keys(), branchRevNum);
- if(exportedCommits.contains(closestCommit))
- {
- bool pathFound = false;
- QString path;
- foreach(path, exportedCommits[closestCommit]) {
- if(path.contains(branchFrom)) {
- pathFound = true;
- break;
- }
- }
-
- if(pathFound) {
- branchFromRef = ":" + QByteArray::number(commitMarks.value(closestCommit));
- qDebug() << branch << "in repository" << name << "is branching from" << closestCommit << "(svn reports r" << branchRevNum << ") git mark:" << branchFromRef;
- } else {
- qWarning() << branch << "in repository" << name << "is branching from a revision that doesn't touch the branch from path, branching from current revision";
- branchFromRef = branchFrom.toUtf8();
- if (!branchFromRef.startsWith("refs/"))
- branchFromRef.prepend("refs/heads/");
- }
- } else {
+ QByteArray branchFromRef = ":" + QByteArray::number(mark);
+ if (!mark) {
qWarning() << branch << "in repository" << name << "is branching but no exported commits exist in repository"
<< "creating an empty branch.";
branchFromRef = branchFrom.toUtf8();
if (!branchFromRef.startsWith("refs/"))
branchFromRef.prepend("refs/heads/");
+ branchFromDesc += ", deleted/unknown";
}
- if (!branches.contains(branchFrom) || !branches.value(branchFrom).created) {
- qCritical() << branch << "in repository" << name
- << "is branching from branch" << branchFrom
- << "but the latter doesn't exist. Can't continue.";
- exit(1);
+ return resetBranch(branch, revnum, mark, branchFromRef, branchFromDesc);
+}
+
+int FastImportRepository::deleteBranch(const QString &branch, int revnum)
+{
+ startFastImport();
+
+ static QByteArray null_sha(40, '0');
+ return resetBranch(branch, revnum, 0, null_sha, "delete");
+}
+
+int FastImportRepository::resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment)
+{
+ QByteArray branchRef = branch.toUtf8();
+ if (!branchRef.startsWith("refs/"))
+ branchRef.prepend("refs/heads/");
+
+ Branch &br = branches[branch];
+ if (br.created && br.created != revnum && br.marks.last()) {
+ QByteArray backupBranch = "refs/backups/r" + QByteArray::number(revnum) + branchRef.mid(4);
+ qWarning() << "backing up branch" << branch << "to" << backupBranch;
+
+ fastImport.write("reset " + backupBranch + "\nfrom " + branchRef + "\n\n");
}
- fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n"
- "progress Branch " + branchRef + " created from "
- + branchFromRef + " r" + QByteArray::number(branchRevNum) + "(at SVN" + QByteArray::number(revnum) + ")\n\n");
+ br.created = revnum;
+ br.commits.append(revnum);
+ br.marks.append(mark);
+
+ fastImport.write("reset " + branchRef + "\nfrom " + resetTo + "\n\n"
+ "progress SVN r" + QByteArray::number(revnum)
+ + " branch " + branch.toUtf8() + " = :" + QByteArray::number(mark)
+ + " # " + comment + "\n\n");
+
+ return EXIT_SUCCESS;
}
-Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix,
+Repository::Transaction *FastImportRepository::newTransaction(const QString &branch, const QString &svnprefix,
int revnum)
{
startFastImport();
@@ -221,7 +550,13 @@ Repository::Transaction *Repository::newTransaction(const QString &branch, const
return txn;
}
-void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix,
+void FastImportRepository::forgetTransaction(Transaction *)
+{
+ if (!--outstandingTransactions)
+ next_file_mark = maxMark;
+}
+
+void FastImportRepository::createAnnotatedTag(const QString &ref, const QString &svnprefix,
int revnum,
const QByteArray &author, uint dt,
const QByteArray &log)
@@ -244,7 +579,7 @@ void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix
tag.dt = dt;
}
-void Repository::finalizeTags()
+void FastImportRepository::finalizeTags()
{
if (annotatedTags.isEmpty())
return;
@@ -291,7 +626,7 @@ void Repository::finalizeTags()
printf("\n");
}
-void Repository::startFastImport()
+void FastImportRepository::startFastImport()
{
if (fastImport.state() == QProcess::NotRunning) {
if (processHasStarted)
@@ -299,15 +634,13 @@ void Repository::startFastImport()
processHasStarted = true;
// start the process
- QString marksFile = name;
- marksFile.replace('/', '_');
- marksFile.prepend("marks-");
+ QString marksFile = marksFileName(name);
QStringList marksOptions;
+ marksOptions << "--import-marks=" + marksFile;
marksOptions << "--export-marks=" + marksFile;
- QString outputFile = name;
- outputFile.replace('/', '_');
- outputFile.prepend("log-");
- fastImport.setStandardOutputFile(outputFile, QIODevice::Append);
+ marksOptions << "--force";
+
+ fastImport.setStandardOutputFile(logFileName(name), QIODevice::Append);
fastImport.setProcessChannelMode(QProcess::MergedChannels);
if (!CommandLineParser::instance()->contains("dry-run")) {
@@ -320,38 +653,60 @@ void Repository::startFastImport()
}
}
-Repository::Transaction::~Transaction()
+FastImportRepository::Transaction::~Transaction()
{
- --repository->outstandingTransactions;
+ repository->forgetTransaction(this);
}
-void Repository::Transaction::setAuthor(const QByteArray &a)
+void FastImportRepository::Transaction::setAuthor(const QByteArray &a)
{
author = a;
}
-void Repository::Transaction::setDateTime(uint dt)
+void FastImportRepository::Transaction::setDateTime(uint dt)
{
datetime = dt;
}
-void Repository::Transaction::setLog(const QByteArray &l)
+void FastImportRepository::Transaction::setLog(const QByteArray &l)
{
log = l;
}
-void Repository::Transaction::deleteFile(const QString &path)
+void FastImportRepository::Transaction::noteCopyFromBranch(const QString &branchFrom, int branchRevNum)
+{
+ static QByteArray dummy;
+ int mark = repository->markFrom(branchFrom, branchRevNum, dummy);
+ Q_ASSERT(dummy.isEmpty());
+
+ if (mark == -1) {
+ qWarning() << branch << "is copying from branch" << branchFrom
+ << "but the latter doesn't exist. Continuing, assuming the files exist.";
+ } else if (mark == 0) {
+ qWarning() << "Unknown revision r" << QByteArray::number(branchRevNum)
+ << ". Continuing, assuming the files exist.";
+ } else {
+ qWarning() << "repository " + repository->name + " branch " + branch + " has some files copied from " + branchFrom + "@" + QByteArray::number(branchRevNum);
+
+ if (!merges.contains(mark))
+ merges.append(mark);
+ }
+}
+
+void FastImportRepository::Transaction::deleteFile(const QString &path)
{
QString pathNoSlash = path;
if(pathNoSlash.endsWith('/'))
pathNoSlash.chop(1);
deletedFiles.append(pathNoSlash);
- modifiedPaths.append(path);
}
-QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length)
+QIODevice *FastImportRepository::Transaction::addFile(const QString &path, int mode, qint64 length)
{
- int mark = ++repository->lastmark;
+ int mark = repository->next_file_mark--;
+
+ // in case the two mark allocations meet, we might as well just abort
+ Q_ASSERT(mark > repository->last_commit_mark + 1);
if (modifiedFiles.capacity() == 0)
modifiedFiles.reserve(2048);
@@ -370,14 +725,22 @@ QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint6
repository->fastImport.write(QByteArray::number(length));
repository->fastImport.write("\n", 1);
}
- modifiedPaths.append(branch + "/" + path);
+
return &repository->fastImport;
}
-void Repository::Transaction::commit()
+void FastImportRepository::Transaction::commit()
{
processCache.touch(repository);
+ // We might be tempted to use the SVN revision number as the fast-import commit mark.
+ // However, a single SVN revision can modify multple branches, and thus lead to multiple
+ // commits in the same repo. So, we need to maintain a separate commit mark counter.
+ int mark = ++repository->last_commit_mark;
+
+ // in case the two mark allocations meet, we might as well just abort
+ Q_ASSERT(mark < repository->next_file_mark - 1);
+
// create the commit message
QByteArray message = log;
if (!message.endsWith('\n'))
@@ -385,6 +748,18 @@ void Repository::Transaction::commit()
if (CommandLineParser::instance()->contains("add-metadata"))
message += "\nsvn path=" + svnprefix + "; revision=" + QByteArray::number(revnum) + "\n";
+ int parentmark = 0;
+ Branch &br = repository->branches[branch];
+ if (br.created) {
+ parentmark = br.marks.last();
+ } else {
+ qWarning() << "Branch" << branch << "in repository" << repository->name << "doesn't exist at revision"
+ << revnum << "-- did you resume from the wrong revision?";
+ br.created = revnum;
+ }
+ br.commits.append(revnum);
+ br.marks.append(mark);
+
{
QByteArray branchRef = branch;
if (!branchRef.startsWith("refs/"))
@@ -392,24 +767,38 @@ void Repository::Transaction::commit()
QTextStream s(&repository->fastImport);
s << "commit " << branchRef << endl;
- s << "mark :" << QByteArray::number(++repository->lastmark) << endl;
- repository->commitMarks.insert(revnum, repository->lastmark);
- repository->exportedCommits.insert(revnum, modifiedPaths);
+ s << "mark :" << QByteArray::number(mark) << endl;
s << "committer " << QString::fromUtf8(author) << ' ' << datetime << " -0000" << endl;
- Branch &br = repository->branches[branch];
- if (!br.created) {
- qWarning() << "Branch" << branch << "in repository" << repository->name << "doesn't exist at revision"
- << revnum << "-- did you resume from the wrong revision?";
- br.created = revnum;
- }
-
s << "data " << message.length() << endl;
}
repository->fastImport.write(message);
repository->fastImport.putChar('\n');
+ // note some of the inferred merges
+ QByteArray desc = "";
+ int i = !!parentmark; // if parentmark != 0, there's at least one parent
+ foreach (int merge, merges) {
+ if (merge == parentmark)
+ continue;
+
+ if (++i > 16) {
+ // FIXME: options:
+ // (1) ignore the 16 parent limit
+ // (2) don't emit more than 16 parents
+ // (3) create another commit on branch to soak up additional parents
+ // we've chosen option (2) for now, since only artificial commits
+ // created by cvs2svn seem to have this issue
+ qWarning() << "too many merge parents";
+ break;
+ }
+
+ QByteArray m = " :" + QByteArray::number(merge);
+ desc += m;
+ repository->fastImport.write("merge" + m + "\n");
+ }
+
// write the file deletions
if (deletedFiles.contains(""))
repository->fastImport.write("deleteall\n");
@@ -420,10 +809,10 @@ void Repository::Transaction::commit()
// write the file modifications
repository->fastImport.write(modifiedFiles);
- repository->fastImport.write("\nprogress Commit #" +
- QByteArray::number(repository->commitCount) +
- " branch " + branch +
- " = SVN r" + QByteArray::number(revnum) + "\n\n");
+ repository->fastImport.write("\nprogress SVN r" + QByteArray::number(revnum)
+ + " branch " + branch + " = :" + QByteArray::number(mark)
+ + (desc.isEmpty() ? "" : " # merge from") + desc
+ + "\n\n");
printf(" %d modifications from SVN %s to %s/%s",
deletedFiles.count() + modifiedFiles.count(), svnprefix.data(),
qPrintable(repository->name), branch.data());
diff --git a/src/repository.h b/src/repository.h
index d008996..e07d184 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -30,78 +30,36 @@ public:
class Transaction
{
Q_DISABLE_COPY(Transaction)
- friend class Repository;
-
- Repository *repository;
- QByteArray branch;
- QByteArray svnprefix;
- QByteArray author;
- QByteArray log;
- uint datetime;
- int revnum;
-
- QStringList deletedFiles;
- QByteArray modifiedFiles;
- QVector<QString> modifiedPaths;
-
- inline Transaction() {}
+ protected:
+ Transaction() {}
public:
- ~Transaction();
- void commit();
+ virtual ~Transaction() {}
+ virtual void commit() = 0;
- void setAuthor(const QByteArray &author);
- void setDateTime(uint dt);
- void setLog(const QByteArray &log);
+ virtual void setAuthor(const QByteArray &author) = 0;
+ virtual void setDateTime(uint dt) = 0;
+ virtual void setLog(const QByteArray &log) = 0;
- void deleteFile(const QString &path);
- QIODevice *addFile(const QString &path, int mode, qint64 length);
- };
- Repository(const Rules::Repository &rule);
- ~Repository();
+ virtual void noteCopyFromBranch (const QString &prevbranch, int revFrom) = 0;
- void reloadBranches();
- void createBranch(const QString &branch, int revnum,
- const QString &branchFrom, int revFrom);
- Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum);
-
- void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum,
- const QByteArray &author, uint dt,
- const QByteArray &log);
- void finalizeTags();
-
-private:
- struct Branch
- {
- int created;
+ virtual void deleteFile(const QString &path) = 0;
+ virtual QIODevice *addFile(const QString &path, int mode, qint64 length) = 0;
};
- struct AnnotatedTag
- {
- QString supportingRef;
- QByteArray svnprefix;
- QByteArray author;
- QByteArray log;
- uint dt;
- int revnum;
- };
-
- QHash<QString, Branch> branches;
- QHash<QString, AnnotatedTag> annotatedTags;
- // rXXXX, mark
- QHash<int, int> commitMarks;
- // rXXXX, [path, path, ...]
- QHash<int, QVector<QString> > exportedCommits;
- QString name;
- QProcess fastImport;
- int commitCount;
- int outstandingTransactions;
- int lastmark;
- bool processHasStarted;
-
- void startFastImport();
- void closeFastImport();
-
- friend class ProcessCache;
- Q_DISABLE_COPY(Repository)
+ virtual int setupIncremental(int &cutoff) = 0;
+ virtual void restoreLog() = 0;
+ virtual ~Repository() {}
+
+ virtual int createBranch(const QString &branch, int revnum,
+ const QString &branchFrom, int revFrom) = 0;
+ virtual int deleteBranch(const QString &branch, int revnum) = 0;
+ virtual Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum) = 0;
+
+ virtual void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum,
+ const QByteArray &author, uint dt,
+ const QByteArray &log) = 0;
+ virtual void finalizeTags() = 0;
};
+Repository *makeRepository(const Rules::Repository &rule, const QHash<QString, Repository *> &repositories);
+
#endif
diff --git a/src/ruleparser.cpp b/src/ruleparser.cpp
index 9a83203..d6e634e 100644
--- a/src/ruleparser.cpp
+++ b/src/ruleparser.cpp
@@ -81,8 +81,19 @@ void Rules::load()
repo.branches += branch;
continue;
- } else if (line == "end repository") {
+ } else if (matchRepoLine.exactMatch(line)) {
+ repo.forwardTo = matchRepoLine.cap(1);
+ continue;
+ } else if (matchPrefixLine.exactMatch(line)) {
+ repo.prefix = matchPrefixLine.cap(1);
+ continue;
+ } else if (line == "end repository") {
m_repositories += repo;
+ {
+ // clear out 'repo'
+ Repository temp;
+ std::swap(repo, temp);
+ }
state = ReadingNone;
continue;
}
diff --git a/src/ruleparser.h b/src/ruleparser.h
index 8d0fcb2..95a508f 100644
--- a/src/ruleparser.h
+++ b/src/ruleparser.h
@@ -36,6 +36,9 @@ public:
QList<Branch> branches;
int lineNumber;
+ QString forwardTo;
+ QString prefix;
+
Repository() : lineNumber(0) { }
};
diff --git a/src/svn.cpp b/src/svn.cpp
index 2a9ffcb..594a750 100644
--- a/src/svn.cpp
+++ b/src/svn.cpp
@@ -512,7 +512,6 @@ int SvnRevision::exportEntry(const char *key, const svn_fs_path_change_t *change
svn_boolean_t is_dir;
SVN_ERR(svn_fs_is_dir(&is_dir, fs_root, key, revpool));
if (is_dir) {
- current += '/';
if (change->change_kind == svn_fs_path_change_modify ||
change->change_kind == svn_fs_path_change_add) {
if (path_from == NULL) {
@@ -523,8 +522,6 @@ int SvnRevision::exportEntry(const char *key, const svn_fs_path_change_t *change
}
qDebug() << " " << key << "was copied from" << path_from << "rev" << rev_from;
- } else if (change->change_kind == svn_fs_path_change_delete) {
- qDebug() << " " << key << "was deleted";
} else if (change->change_kind == svn_fs_path_change_replace) {
if (path_from == NULL)
qDebug() << " " << key << "was replaced";
@@ -534,11 +531,17 @@ int SvnRevision::exportEntry(const char *key, const svn_fs_path_change_t *change
qCritical() << " " << key << "was reset, panic!";
return EXIT_FAILURE;
} else {
+ // if change_kind == delete, it shouldn't come into this arm of the 'is_dir' test
qCritical() << " " << key << "has unhandled change kind " << change->change_kind << ", panic!";
return EXIT_FAILURE;
}
+ } else if (change->change_kind == svn_fs_path_change_delete) {
+ is_dir = wasDir(fs, revnum - 1, key, revpool);
}
+ if (is_dir)
+ current += '/';
+
// find the first rule that matches this pathname
MatchRuleList::ConstIterator match = findMatchRule(matchRules, revnum, current);
if (match != matchRules.constEnd()) {
@@ -549,6 +552,9 @@ int SvnRevision::exportEntry(const char *key, const svn_fs_path_change_t *change
if (is_dir && path_from != NULL) {
qDebug() << current << "is a copy-with-history, auto-recursing";
return recurse(key, change, path_from, rev_from, changes, revpool);
+ } else if (is_dir && change->change_kind == svn_fs_path_change_delete) {
+ qDebug() << current << "deleted, auto-recursing";
+ return recurse(key, change, path_from, rev_from, changes, revpool);
} else if (wasDir(fs, revnum - 1, key, revpool)) {
qDebug() << current << "was a directory; ignoring";
} else if (change->change_kind == svn_fs_path_change_delete) {
@@ -579,7 +585,14 @@ int SvnRevision::exportDispatch(const char *key, const svn_fs_path_change_t *cha
return recurse(key, change, path_from, rev_from, changes, pool);
case Rules::Match::Export:
- return exportInternal(key, change, path_from, rev_from, current, rule);
+ if (exportInternal(key, change, path_from, rev_from, current, rule) == EXIT_SUCCESS)
+ return EXIT_SUCCESS;
+ if (change->change_kind != svn_fs_path_change_delete)
+ return EXIT_FAILURE;
+ // we know that the default action inside recurse is to recurse further or to ignore,
+ // either of which is reasonably safe for deletion
+ qWarning() << "deleting unknown path" << current << "; auto-recursing";
+ return recurse(key, change, path_from, rev_from, changes, pool);
}
// never reached
@@ -593,72 +606,86 @@ int SvnRevision::exportInternal(const char *key, const svn_fs_path_change_t *cha
QString svnprefix, repository, branch, path;
splitPathName(rule, current, &svnprefix, &repository, &branch, &path);
+ Repository *repo = repositories.value(repository, 0);
+ if (!repo) {
+ if (change->change_kind != svn_fs_path_change_delete)
+ qCritical() << "Rule" << rule
+ << "references unknown repository" << repository;
+ return EXIT_FAILURE;
+ }
+
printf(".");
fflush(stdout);
// qDebug() << " " << qPrintable(current) << "rev" << revnum << "->"
// << qPrintable(repository) << qPrintable(branch) << qPrintable(path);
- if (path.isEmpty() && path_from != NULL) {
- QString previous = QString::fromUtf8(path_from) + '/';
+ if (change->change_kind == svn_fs_path_change_delete && current == svnprefix) {
+ qDebug() << "repository" << repository << "branch" << branch << "deleted";
+ return repo->deleteBranch(branch, revnum);
+ }
+
+ QString previous;
+ QString prevsvnprefix, prevrepository, prevbranch, prevpath;
+
+ if (path_from != NULL) {
+ previous = QString::fromUtf8(path_from) + '/';
MatchRuleList::ConstIterator prevmatch =
findMatchRule(matchRules, rev_from, previous, NoIgnoreRule);
- if (prevmatch != matchRules.constEnd()) {
- QString prevsvnprefix, prevrepository, prevbranch, prevpath;
+ if (prevmatch != matchRules.constEnd())
splitPathName(*prevmatch, previous, &prevsvnprefix, &prevrepository,
&prevbranch, &prevpath);
-
- if (!prevpath.isEmpty()) {
- qDebug() << qPrintable(current) << "is a partial branch of repository"
- << qPrintable(prevrepository) << "branch"
- << qPrintable(prevbranch) << "subdir"
- << qPrintable(prevpath);
- } else if (prevrepository != repository) {
- qWarning() << qPrintable(current) << "rev" << revnum
- << "is a cross-repository copy (from repository"
- << qPrintable(prevrepository) << "branch"
- << qPrintable(prevbranch) << "path"
- << qPrintable(prevpath) << "rev" << rev_from << ")";
- } else if (prevbranch == branch) {
- // same branch and same repository
- qDebug() << qPrintable(current) << "rev" << revnum
- << "is an SVN rename from"
- << qPrintable(previous) << "rev" << rev_from;
- return EXIT_SUCCESS;
- } else {
- // same repository but not same branch
- // this means this is a plain branch
- qDebug() << qPrintable(repository) << ": branch"
- << qPrintable(branch) << "is branching from"
- << qPrintable(prevbranch);
-
- Repository *repo = repositories.value(repository, 0);
- if (!repo) {
- qCritical() << "Rule" << rule
- << "references unknown repository" << repository;
- return EXIT_FAILURE;
- }
-
- repo->createBranch(branch, revnum, prevbranch, rev_from);
- if (rule.annotate) {
- // create an annotated tag
- fetchRevProps();
- repo->createAnnotatedTag(branch, svnprefix, revnum, authorident,
- epoch, log);
- }
- return EXIT_SUCCESS;
- }
- }
+ else
+ path_from = NULL;
}
+ // current == svnprefix => we're dealing with the contents of the whole branch here
+ if (path_from != NULL && current == svnprefix) {
+ if (previous != prevsvnprefix) {
+ // source is not the whole of its branch
+ qDebug() << qPrintable(current) << "is a partial branch of repository"
+ << qPrintable(prevrepository) << "branch"
+ << qPrintable(prevbranch) << "subdir"
+ << qPrintable(prevpath);
+ } else if (prevrepository != repository) {
+ qWarning() << qPrintable(current) << "rev" << revnum
+ << "is a cross-repository copy (from repository"
+ << qPrintable(prevrepository) << "branch"
+ << qPrintable(prevbranch) << "path"
+ << qPrintable(prevpath) << "rev" << rev_from << ")";
+ } else if (path != prevpath) {
+ qDebug() << qPrintable(current)
+ << "is a branch copy which renames base directory of all contents"
+ << qPrintable(prevpath) << "to" << qPrintable(path);
+ // FIXME: Handle with fast-import 'file rename' facility
+ // ??? Might need special handling when path == / or prevpath == /
+ } else {
+ if (prevbranch == branch) {
+ // same branch and same repository
+ qDebug() << qPrintable(current) << "rev" << revnum
+ << "is reseating branch" << qPrintable(branch)
+ << "to an earlier revision"
+ << qPrintable(previous) << "rev" << rev_from;
+ } else {
+ // same repository but not same branch
+ // this means this is a plain branch
+ qDebug() << qPrintable(repository) << ": branch"
+ << qPrintable(branch) << "is branching from"
+ << qPrintable(prevbranch);
+ }
+
+ if (repo->createBranch(branch, revnum, prevbranch, rev_from) == EXIT_FAILURE)
+ return EXIT_FAILURE;
+ if (rule.annotate) {
+ // create an annotated tag
+ fetchRevProps();
+ repo->createAnnotatedTag(branch, svnprefix, revnum, authorident,
+ epoch, log);
+ }
+ return EXIT_SUCCESS;
+ }
+ }
Repository::Transaction *txn = transactions.value(repository + branch, 0);
if (!txn) {
- Repository *repo = repositories.value(repository, 0);
- if (!repo) {
- qCritical() << "Rule" << rule
- << "references unknown repository" << repository;
- return EXIT_FAILURE;
- }
-
txn = repo->newTransaction(branch, svnprefix, revnum);
if (!txn)
return EXIT_FAILURE;
@@ -666,6 +693,15 @@ int SvnRevision::exportInternal(const char *key, const svn_fs_path_change_t *cha
transactions.insert(repository + branch, txn);
}
+ //
+ // If this path was copied from elsewhere, use it to infer _some_
+ // merge points. This heuristic is fairly useful for tracking
+ // changes across directory re-organizations and wholesale branch
+ // imports.
+ //
+ if (path_from != NULL && prevrepository == repository)
+ txn->noteCopyFromBranch (prevbranch, rev_from);
+
if (change->change_kind == svn_fs_path_change_replace && path_from == NULL)
txn->deleteFile(path);
if (change->change_kind == svn_fs_path_change_delete) {
@@ -684,6 +720,10 @@ int SvnRevision::recurse(const char *path, const svn_fs_path_change_t *change,
const char *path_from, svn_revnum_t rev_from,
apr_hash_t *changes, apr_pool_t *pool)
{
+ svn_fs_root_t *fs_root = this->fs_root;
+ if (change->change_kind == svn_fs_path_change_delete)
+ SVN_ERR(svn_fs_revision_root(&fs_root, fs, revnum - 1, pool));
+
// get the dir listing
apr_hash_t *entries;
SVN_ERR(svn_fs_dir_entries(&entries, fs_root, path, pool));
@@ -724,9 +764,11 @@ int SvnRevision::recurse(const char *path, const svn_fs_path_change_t *change,
rev_from, changes, current, *match, dirpool) == EXIT_FAILURE)
return EXIT_FAILURE;
} else {
- qCritical() << current << "rev" << revnum
- << "did not match any rules; cannot continue";
- return EXIT_FAILURE;
+ qDebug() << current << "rev" << revnum
+ << "did not match any rules; auto-recursing";
+ if (recurse(entry, change, entryFrom.isNull() ? 0 : entryFrom.constData(),
+ rev_from, changes, dirpool) == EXIT_FAILURE)
+ return EXIT_FAILURE;
}
}
generated by cgit v1.2.1 (git 2.21.0) at 2024-10-02 01:11:18 +0000