use urpmi -nolock

log chroot initialisation
change logging mode for external command to correctly get the exit code
try to catch out of space problem with unionfs and recompile the package in mormal chrooted mode
fix needed structure bad stored version require
initialise chroot before chroot initialisation in nocheckroot mode

1 files changed, 94 insertions, 39 deletions

diff --git a/iurt2 b/iurt2
index 3ada7c9..7879208 100755
--- a/iurt2
+++ b/iurt2
@@ -24,6 +24,7 @@
 # TODO
 #
 # - use a cache (rpmctl cache for example) to find maintainer
+# - Do not run if the hdlist are broken
 #
 use strict;
 use Hdlist;
@@ -48,7 +49,7 @@ $distro_tag =~ s,/,-,g;
 my $my_arch = shift @argv;
 my $media = shift @argv;
 my @special_srpm_dir = @argv;
-my $urpmi_options = "-v --no-verify-rpm -s --auto";
+my $urpmi_options = "-v --no-verify-rpm --nolock --auto";
 
 $my_arch or usage("no architecture given");
 my $real_arch = `uname -m`;
@@ -141,7 +142,7 @@ my $cache;
 if (-f $cachefile) {
     $cache = do $cachefile
 } else {
-    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {} }
+    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {}, no_unionfs => {} }
 }
 
 my %big;
@@ -247,7 +248,6 @@ foreach my $dir ("$config->{repository}/$distro_version/SRPMS/$media/", @special
 	closedir $rpmdir
 }
 
-
 dump_cache();
 
 if (!@todo && !$debug) {
@@ -260,6 +260,14 @@ print "iurt: will try to compile $to_compile packages\n";
 $cache->{run} ||= 1;
 my $run = ++$cache->{run};
 
+my $local_spool = "$config->{local_home}/iurt/$distro_tag/$my_arch";
+if (!-d "$config->{local_home}/iurt/$distro_tag/") {
+    mkdir "$config->{local_home}/iurt/$distro_tag";
+    if (!-d $local_spool) {
+	mkdir $local_spool;
+	mkdir "$local_spool/log"
+    }
+}
 
 my $debug_tag = '_debug' if $debug;
 my $chroot = "$config->{local_home}/chroot$debug_tag";
@@ -273,6 +281,7 @@ if (!$nocheckchroot) {
 	error => "[REBUILD] Creating the inital chroot for $distro_tag on $my_arch failed", 
 	hash => 'chroot_inititialization', 
 	timeout => 600, 
+	log => "$local_spool/log/", 
 	debug_mail => $debug,
 	die => 1);
 
@@ -286,16 +295,6 @@ if ($unionfs) {
     $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag";
     -d $unionfs_dir or mkdir $unionfs_dir
 }
-
-my $local_spool = "$config->{local_home}/iurt/$distro_tag/$my_arch";
-if (!-d "$config->{local_home}/iurt/$distro_tag/") {
-    mkdir "$config->{local_home}/iurt/$distro_tag";
-    if (!-d $local_spool) {
-	mkdir $local_spool;
-	mkdir "$local_spool/log"
-    }
-}
-
 my $s = sub { print "iurt: dumping cache...\n"; dump_cache(); exit };
 $SIG{TERM} = $s;
 $SIG{INT} = $s;
@@ -313,8 +312,28 @@ foreach (my $i ; $i < @todo; $i++) {
 	if ($debug) { $debug++ == 2 and exit }
 	$done++;
 	print "iurt: packages $srpm [$done/$to_compile]\n";
-	if ($unionfs) {
+	# FIXME unfortunately urpmi stalls quite often
+	my $match = "urpmi $urpmi_options --root $chroot";
+	if (!clean_process($match)) {
+	    dump_cache();
+	    die "FATAL iurt: Could not have urpmi working !"
+	}
+	my $unionfs_tmp;	
+	if ($cache->{no_unionfs}{$srpm}) {
+	    $unionfs_tmp = 0
+	} elsif ($unionfs) {
+	    $unionfs_tmp = 1
+	}
+	if ($unionfs_tmp) {
+	    my $mount_point = "$unionfs_dir/unionfs.$run.$union_id";
+	    print STDERR "Cleaning $mount_point\n";
+	    if (!clean_mnt($mount_point)) {
+		dump_cache();
+		die "FATAL iurt: could not kill remaining processes acceding $mount_point"
+	    }
 	    my $tmpfs;
+	    
+	    # we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database)
 	    $union_id = clean_unionfs($unionfs_dir, $run, $union_id);
 	    $tmpfs = "$unionfs_dir/tmpfs.$run.$union_id";
 	    $chroot = "$unionfs_dir/unionfs.$run.$union_id";
@@ -328,19 +347,15 @@ foreach (my $i ; $i < @todo; $i++) {
 	}
 	my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
 	my $maintainer = `rpmmon -s -p $srpm_name`;
-	my $cc = "$maint{$srpm}, maintainers\@mandriva.com";
+	my $cc = "$maint{$srpm}";#, maintainers\@mandriva.com";
 	chomp $maintainer;
 	if (!$maintainer || $maintainer eq 'NOT_FOUND') {
 		$maintainer = $cc;
-		$cc = 'maintainers@mandriva.com'
+		#$cc = 'maintainers@mandriva.com'
 	}
 	#($maintainer, $cc) = ($config->{admin},'');
 	print "Installing build dependencies of $srpm...\n";
-	# FIXME unfortunately urpmi stalls quite often
-	if (!clean_urpmi($home)) {
-	    dump_cache();
-	    die "FATAL iurt: Could not have urpmi working !"
-	}
+	
 	#system(qq{sudo pkill -9 dd-u root -f "$todo[$i-1][1]"}) if $i - 1 >= 0;
 	#system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot"});
 	perform_command("sudo urpmi $urpmi_options --root $chroot $dir/$srpm", 
@@ -357,17 +372,17 @@ foreach (my $i ; $i < @todo; $i++) {
 		log => "$local_spool/log/",
 		callback => sub { 
 			my ($opt, $output) = @_;
-			print "Calling callback for $opt->{hash}\n" if $debug;
-			my ($missing_deps, $version) = $output =~ /\(due to unsatisfied ([^[)]*)(\[.*\])?/;
+			print "calling callback for $opt->{hash}\n" if $debug;
+			my ($missing_deps, $version) = $output =~ /\(due to unsatisfied ([^[)]*)(?:\[(.*)\])?/;
 			$missing_deps or return;
 			my $p = $pack_provide{$missing_deps} || $missing_deps;
 			my $other_maint = `rpmmon -p $p`;
-			$version ||= 1;
+			$version ||= 0;
 			chomp $other_maint;
-			print "Missing Dep: $missing_deps ($other_maint)\n";
-			if ($other_maint && $other_maint ne 'NOT_FOUND') {
+			print "missing dep: $missing_deps ($other_maint)\n";
+			if ($other_maint && $other_maint ne 'not_found') {
 			    $opt->{mail} = $other_maint;
-			    $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $my_arch";
+			    $opt->{error} = "[missing] $missing_deps, needed to build $srpm, is not available on $my_arch";
 			}
 			# remember what is needed, and do not try to recompile until it is available
 			push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
@@ -395,8 +410,32 @@ foreach (my $i ; $i < @todo; $i++) {
 			cc => $cc, 
 			log => "$local_spool/log/", 
 			error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
+			callback => sub { 
+			    my ($opt, $output) = @_;
+			    print STDERR "iurt: calling callback for $opt->{hash}\n" if $debug;
+			    if ($unionfs_tmp && $output =~ /no space left on device/i) {
+				print STDERR "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
+				$cache->{no_unionfs}{$srpm} = 1
+			    }
+			}, 
 			freq => 1) && !glob "$chroot/home/builder/rpm/RPMS/*/*.rpm") {
-		$cache->{failure}{$srpm} = 1;
+		    # FIXME
+		    # The simple algo used here is : 
+		    #  try to compile it with unionfs, if it runs out of space, compile it without the next time
+		    #
+		    #  This could be improved in keeping this srpm name for future version, but if we compile it 
+		    #  on a new machine with more ram, or if next version compiles just fine with unionfs, we will
+		    #  loose the unionfs advantage. 
+		    #
+		    #  Maybe the right thing to do would be to first try to increate the tmpfs size (more than 50 % of the 
+		    #  physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs
+		    #  plus). Or to keep the faulty package an unionfs exception for some time, to save some more extra builds.
+		    # 
+		    if (!$unionfs_tmp) {
+			$cache->{failure}{$srpm} = 1
+		    } elsif ($cache->{no_unionfs}{$srpm}) {
+			$cache->{failure}{$srpm} = 1
+		    }
 		next
 	}
 	# do some cleaning if the compilation is successful
@@ -496,18 +535,26 @@ sub clean_chroot {
    1
 }
 
-sub clean_urpmi {
-    my ($chroot) = @_;
-    my $match = "urpmi $urpmi_options --root $chroot";
+sub clean_process {
+    my ($match) = @_;
+    return clean($match, "pgrep -u root -f", "sudo pkill -9 -u root -f");
+}
+
+sub clean_mnt {
+    my ($mount_point) = @_;
+    return clean($mount_point, "fuser", "sudo fuser -k");
+}
+
+sub clean {
+    my ($var, $cmd, $kill_cmd) = @_;
     my $ps;
     my $i;
-    $ps = `pgrep -u root -f "$match"`;
-    while ($ps) {
-	system(qq{sudo pkill -9 -u root -f "$match"});
+    while ($ps = `$cmd "$var"`) {
+	system(qq{$kill_cmd "$var"});
 	sleep 1;
-	$ps = `pgrep -u root -f "$match"`;
-	print STDERR "Trying to removed previous blocked urpmi session matching $match ($ps)\n";
-	$i++ > 10 and return 0
+	chomp $ps;
+	print STDERR "Trying to removed previous blocked processes for $var ($ps)\n";
+	return 0 if $i++ > 10
     }
     1
 }
@@ -557,6 +604,8 @@ sub check_needed {
 	}
 	$ok = 0;
 	push @n, [ $name, $version ];
+	my $v ||= $version;
+	print STDERR "ERROR iurt: $srpm needs $name $v to be compiled.\n";
 	# try to recompile it once in a while
 	return 1 if ! $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72
     }
@@ -675,7 +724,8 @@ sub perform_command {
 	alarm $opt{timeout};
 	print "$command\n";
 	if ($opt{log}) {
-	    $output = `$command 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
+	    #$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
+	    system("$command &> $opt{log}/$opt{hash}.$run.log");
 	} else {
 	    $output = `$command 2>&1`;
 	}
@@ -686,6 +736,11 @@ sub perform_command {
 	die unless $@ eq "alarm\n";   # propagate unexpected errors
 	return 0
     } else {
+	# Keep the run first on the harddrive so that one can check the command status tailing it
+	if ($opt{log} && open my $log, "$opt{log}/$opt{hash}.$run.log") {
+	    local $/;
+	    $output = <$log>
+	}
 	my $err = $?;
 	if ($kill) {
 	    $output = "Command has been killed after $opt{timeout} seconds: $command\n$output"
@@ -707,7 +762,7 @@ sub perform_command {
 	}
 	if ($err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) {
 	    if ($opt{mail} && $config->{sendmail}) {
-		if (! $cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq}) {
+		if (! ($cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq})) {
 		    sendmail($opt{mail}, $opt{cc} , $opt{error} , $output, 0, 0, $opt{debug_mail});
 		} elsif ($config->{admin}) {
 		    sendmail($config->{admin}, '' , $opt{error}, $output, 0, 0, $opt{debug_mail});