flock and lockf does not work correctly, try to workarround

1 files changed, 81 insertions, 30 deletions

diff --git a/iurt2 b/iurt2
index 6fcc32e..91cd1ab 100755
--- a/iurt2
+++ b/iurt2
@@ -34,7 +34,8 @@ use Data::Dumper;
 use URPM;
 use File::NCopy qw(copy);
 use MIME::Words qw(encode_mimewords);
-use Fcntl ':flock';
+# I did not manage to make locks work over the network
+#use File::lockf;
 use Mkcd::Commandline qw(parseCommandLine usage);
 use Filesys::Df qw(df);
 
@@ -246,6 +247,9 @@ if ($run{distro} ne 'cooker') {
 
 -d $config->{upload} or usage($program_name, \@params, "$config->{upload} does not exist");
 
+# cache file name is needed early to remove the manual lock file if the lock mechanism does not work
+my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$run{media}.cache";
+$run{cachefile} = $cachefile;
 if (!$run{debug} && $run{media} || $run{chroot}) {
     $run{pidfile_home} = "$config->{cache_home}/";
     $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$run{media}";
@@ -259,8 +263,6 @@ if ($run{chroot}) {
     check_chroot($chroot, $chroot_tar, \%run)
 }
 
-my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$run{media}.cache";
-$run{cachefile} = $cachefile;
 my $cache;
 my $clear_cache = 1;
 if (-f $cachefile && $run{use_cache}) {
@@ -394,6 +396,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 	mkdir "$local_spool/log/$srpm";
 	print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n";
 	# FIXME unfortunately urpmi stalls quite often
+	my $retry = 0;
 retry:
 	my $match = "urpmi $urpmi_options --root $chroot_tmp";
 	if (!clean_process($match, $run{verbose})) {
@@ -451,12 +454,30 @@ retry:
 		error => "[REBUILD] cannot copy $srpm to $chroot_tmp", 
 		debug_mail => $run{debug},
 		hash => "copy_$srpm") or next;
-	perform_command(qq{sudo chroot $chroot_tmp su builder -c "rpm -i /home/builder/rpm/SRPMS/$srpm"}, 
-		\%run, $config,
-		mail => $config->{admin}, 
+
+	my %opt = ( mail => $config->{admin}, 
 		error => "[REBUILD] cannot install $srpm in $chroot_tmp", 
 		debug_mail => $run{debug},
-		hash => "install_$srpm") or next;
+		hash => "install_$srpm",
+		callback => sub { 
+			my ($opt, $output) = @_;
+			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
+			if ($output =~ /user builder does not exist/) {
+			    print {$run{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run{verbose} > 1;
+			    $opt->{error} = "[CHROOT] chroot is corrupted";
+			    $opt->{retry} = 1 if !$retry
+			}
+		    }
+		);
+	if (!perform_command(qq{sudo chroot $chroot_tmp su builder -c "rpm -i /home/builder/rpm/SRPMS/$srpm"}, 
+		\%run, $config, %opt)) {
+		if ($opt{retry}) {
+		    $retry = 1;
+		    check_chroot($chroot, $chroot_tar, \%run);
+		    goto retry
+		}
+		next
+	    }
 	perform_command(qq{sudo chroot $chroot_tmp su builder -c "rpm --nodeps -bs /home/builder/rpm/SPECS/*.spec"}, 
 		\%run, $config,
 		mail => $config->{admin}, 
@@ -839,30 +860,41 @@ sub dump_cache {
     # but the idea is more to have a global parrallel build than several local ones.
     return if $run->{debug} || !$run->{use_cache};
     open my $file, ">$filename.tmp.$daydate" or die "FATAL iurt dump_cache: cannot open $filename.tmp";
-	#seek($file, 0, 2);
-	if ($run{concurrent_run}) {
-	    print {$run{log}} "iurt: merging cache";
-	    my $old_cache;
-	    if (-f $filename) {
-		print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
-		$old_cache = do $cachefile;
-		foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning') {
-		    foreach my $rpm (%{$old_cache->{$k}}) {
-			$cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm}
-		    }
+    if ($run{concurrent_run}) {
+	print {$run{log}} "iurt: merging cache";
+	my $old_cache;
+	if (-f $filename) {
+	    print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
+	    $old_cache = do $cachefile;
+	    foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning') {
+		foreach my $rpm (%{$old_cache->{$k}}) {
+		    $cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm}
 		}
 	    }
-	    #  $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
 	}
-	$Data::Dumper::Indent = 1;
-	$Data::Dumper::Terse = 1;
-	print $file Data::Dumper->Dump([ $cache ], [ "cache" ]);
-    if (flock($file,LOCK_EX)) {
+	#  $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
+    }
+    $Data::Dumper::Indent = 1;
+    $Data::Dumper::Terse = 1;
+    print $file Data::Dumper->Dump([ $cache ], [ "cache" ]);
+    # flock does not work on network files and lockf seems to fail too
+    my $status = 1; #File::lockf::lock($file);
+    if (!$status) {
 	unlink $filename;
 	link "$filename.tmp.$daydate", $filename;
-	flock($file,LOCK_UN)
+	File::lockf::ulock($file)
     } else {
-	print {$run{LOG}} "iurt: dumping_cache failed (could not lock cache file $cachefile $!)";
+	print {$run{LOG}} "WARNING iurt: locking the cache file $cachefile failed (status $status $!), try to lock manually\n";
+	if (-f "$filename.lock") {
+	    print {$run{LOG}} "ERROR iurt: manual file lock exist, do not save the cache\n";
+	} else {
+	    open my $lock, "$filename.lock";
+	    print $lock, $$;
+	    close $lock;
+	    unlink $filename;
+	    link "$filename.tmp.$daydate", $filename;
+	    unlink "$filename.lock"
+	}
     }
 }
 
@@ -1082,8 +1114,23 @@ sub check_pid {
     my $hostname = `hostname`;
     chomp $hostname;
     my $pidfile = $run->{pidfile};
-    open my $lock, "$run->{pidfile_home}/$pidfile.$hostname.pid.lock";
-    flock($lock,LOCK_EX);
+    my $lockfile = "$run->{pidfile_home}/$pidfile$hostname.pid.lock";
+    print {$run->{LOG}} "iurt: trying to lock $lockfile\n";
+    open my $lock, ">$lockfile";
+    my $lock_ok;
+    # lockf seems not to work
+    my $status = 1; #File::lockf::lock($lock);
+    if (!$status) {
+	$lock_ok = 1;
+    } else {
+	print {$run->{LOG}} "ERROR iurt: could not lock pid file (status $status $!)\n";
+	if (! -f "$lockfile.2") {
+	    open my $lock2, ">$lockfile.2";
+	    close $lock2;
+	} else {
+	    die "FATAL iurt: could not lock pid file (status $status $!)\n";
+	}
+    }
     if (!$run->{concurrent_run}) {
 	opendir my $dir, $run->{pidfile_home};
 	foreach my $f (readdir $dir) {
@@ -1136,13 +1183,17 @@ sub check_pid {
 	    unlink $pidfile
 	}
     }
-    print "iurt: setting $pidfile pid lock\n";
+    print {$run->{LOG}} "iurt: setting $pidfile pid lock\n";
     open my $PID, ">$pidfile" or die "FATAL iurt: could not open pidfile $pidfile for writing";
     print $PID $$;
     close $PID;
-
-    flock($lock,LOCK_UN);
+    if ($lock_ok) { 
+	File::lockf::ulock($lock);
+    } else {
+	unlink "$lockfile.2"
+    }
     close $lock;
+    unlink $lockfile
 }
 
 sub check_media {