- add --unionfs option to use tmpfs+unionfs to build the packages

- add --nocheckchroot to directly launch the build without initial basystem tar check
- use a different cache per architecture not to have to handle cache merging

1 files changed, 182 insertions, 68 deletions

diff --git a/iurt2 b/iurt2
index 973a342..ad78503 100755
--- a/iurt2
+++ b/iurt2
@@ -21,6 +21,12 @@
 #
 # compare and rebuild packages on different architecture
 #
+# TODO
+#
+# - use a directory + tmpfs via unionfs to build the packages
+# - having only one cache for all the architecture would need a merging before the cache is dumped, this is a bit overkill, so several cache are used, as a consequence the rpm/srpm structure is duplicated. Maybe a separated cache could be done to handle this structure, or just have it created separately. Anyway this is just a mater of some hundreds of kilobytes and directory listings. 
+# - use a cache (rpmctl cache for example) to find maintainer
+#
 use strict;
 use Hdlist;
 use Data::Dumper;
@@ -29,9 +35,11 @@ use File::NCopy qw(copy);
 use MIME::Words qw(encode_mimewords);
 use Fcntl ':flock';
 
-my ($debug);
+my ($debug, $unionfs, $nocheckchroot);
 my @argv = grep { 
    if (/--debug/) { $debug = 1; 0 }
+   elsif (/--unionfs/) { $unionfs = 1; 0 }
+   elsif (/--nocheckchroot/) { $nocheckchroot = 1; 0 }
    else { 1 }
 } @ARGV;
 
@@ -41,6 +49,7 @@ $distro_tag =~ s,/,-,g;
 my $my_arch = shift @argv;
 my $media = shift @argv;
 my @special_srpm_dir = @argv;
+my $urpmi_options = "-v --no-verify-rpm -s --auto ";
 
 $my_arch or usage();
 my $real_arch = `uname -m`;
@@ -58,7 +67,7 @@ if (!$arch_comp{$real_arch}{$my_arch}) {
 
 my $HOME = $ENV{HOME};
 my $configfile = "$HOME/.iurt.$distro_tag.conf";
-print "iurt: loading config file $configfile\n";
+print STDERR "iurt: loading config file $configfile\n";
 my $config;
 if (-f $configfile) {
     $config = do $configfile or die "FATAL iurt: syntax error in $configfile";
@@ -85,7 +94,7 @@ if ($distro_version ne 'cooker') {
 	$config->{upload} .= "/$media"
     }
 } elsif ($media eq 'contrib') {
-    $config->{upload} = "$config->{upload}/contrib"
+    $config->{upload} =~ s/cooker/contrib/g;
 }
 
 -d $config->{upload} or usage("$config->{upload} does not exist");
@@ -102,7 +111,7 @@ if (!$debug) {
 	if ($pid && getpgrp $pid != -1) {
 	    my $time = $stat[9];
 	    if ($time < time - 36000) {
-		print "iurt: an other iurt pid $pid is running for a very long time, killing it\n";
+		print STDERR "iurt: an other iurt pid $pid is running for a very long time, killing it\n";
 		my $i;
 		while ($i < 5 && getpgrp $pid != -1) {
 		    kill_for_good($pid);
@@ -110,11 +119,11 @@ if (!$debug) {
 		    sleep 1
 		}
 	    } else  {
-		print "iurt: an other iurt is running for $my_arch, pid $pid, since ",time - $time," seconds\n";
+		print STDERR "iurt: an other iurt is running for $my_arch, pid $pid, since ",time - $time," seconds\n";
 		# exit
 	    }
 	} else {
-	    print "iurt: a previous iurt for $my_arch seems dead, cleaning.\n";
+	    print STDERR "iurt: a previous iurt for $my_arch seems dead, cleaning.\n";
 	    unlink $pidfile
 	}
     }
@@ -126,13 +135,13 @@ if (!$debug) {
     close $lock;
 }
 
-my $cachefile = "$config->{cache_home}/iurt.$distro_tag.cache";
-print "iurt: loading cache file $cachefile\n";
+my $cachefile = "$config->{cache_home}/iurt.$distro_tag.$my_arch.cache";
+print STDERR "iurt: loading cache file $cachefile\n";
 my $cache;
 if (-f $cachefile) {
     $cache = do $cachefile
 } else {
-    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => {} }
+    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {} }
 }
 
 my %big;
@@ -142,18 +151,35 @@ my @wrong_rpm;
 # in that case, @supported_arch may be removed
 #
 #foreach my $arch (@{$config->{supported_arch}}) {
-	my $rpms_dir = "$config->{repository}/$distro_version/$my_arch/media/$media/";
-	print "iurt: checking current packages in $rpms_dir\n";
-	opendir my $rpmdir, $rpms_dir or die "Could not open $rpms_dir: $!"; 
-	foreach my $rpm (readdir $rpmdir) {
-		my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm);
-		$rarch or next;
-		$big{$rpm} = 1;
-		$cache->{queue}{$srpm}{$my_arch} = 1;
-		$cache->{queue}{$srpm}{$rarch} = 1;
-		check_version($srpm)
-	}
-	closedir $rpmdir;
+my $rpms_dir = "$config->{repository}/$distro_version/$my_arch/media/$media/";
+print STDERR "iurt: checking current packages in $rpms_dir\n";
+opendir my $rpmdir, $rpms_dir or die "Could not open $rpms_dir: $!"; 
+foreach my $rpm (readdir $rpmdir) {
+    my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm);
+    $rarch or next;
+    $big{$rpm} = 1;
+    $cache->{queue}{$srpm} = 1;
+    check_version($srpm)
+}
+closedir $rpmdir;
+
+my %provides;
+my $synthesis_file = "$config->{repository}/$distro_version/$my_arch/media/$media/media_info/synthesis.hdlist.cz";
+if (-f $synthesis_file) {
+    print STDERR "Parsing $synthesis_file\n";
+    if (open my $syn, "zcat $synthesis_file |") { 
+	while (<$syn>) {
+	    if (/^\@provides@(.*)/) {
+		foreach my $p (split '@', $1) {
+		    $p =~ /([^[]+)(?:\[(.*)\])?/g;
+		    $provides{$1} = $2 || 1
+		}
+	    }
+	}	    
+    } else {
+	print STDERR "ERROR: Could not open $synthesis_file, will not precheck dependecies avaibility\n";
+    }
+}
 #}
 
 my %maint;
@@ -167,6 +193,7 @@ my %done_rpm;
 # FIXME all the rep but the first one are cleaned
 # this may not be usefull anymore
 #
+my $to_compile;
 foreach my $dir ("$config->{repository}/$distro_version/SRPMS/$media/", @special_srpm_dir) {
 	print "iurt: checking SRPMS dir $dir\n";
 	opendir my $rpmdir, $dir or next;
@@ -181,18 +208,19 @@ foreach my $dir ("$config->{repository}/$distro_version/SRPMS/$media/", @special
 		if ($config->{unwanted_packages} && $srpm =~ /$config->{unwanted_packages}/) { next }
 		my $ok = 1;
 		if (check_version($srpm)) { 
-			defined $cache->{failure}{$srpm} && defined $cache->{failure}{$srpm}{$my_arch} and next;
-			if (!$cache->{queue}{$srpm}{$my_arch} && !$cache->{queue}{$srpm}{noarch}) {
+			defined $cache->{failure}{$srpm} and next;
+			if (!$cache->{queue}{$srpm} && check_needed($srpm)) {
 				my $hdr = rpm2header("$dir/$srpm");
 				check_arch($hdr) and next;
 				my $changelog = $hdr->queryformat("%{CHANGELOGNAME}");
 				my ($mail) = $changelog =~ /<(.*@.*)>/;
 				$maint{$srpm} = $mail;
 				print "iurt: will try to compile $srpm\n";
+				$to_compile++;
 				push @todo, [ $dir , $srpm ]
 			}
 			foreach my $arch (@{$config->{supported_arch}}) {
-				$ok &&= $cache->{queue}{$srpm}{$arch} || $cache->{queue}{$srpm}{noarch}
+				$ok &&= $cache->{queue}{$srpm}
 			}
 		}
 		if ($clean && ($rep{$srpm} || $ok)) {
@@ -214,54 +242,70 @@ if (!@todo && !$debug) {
     unlink $pidfile;
     exit
 }
+print "iurt: will try to compile $to_compile packages\n";
 
-my $run = $cache->{run}{$distro_tag}{$my_arch}++;
+my $run = $cache->{run}++;
 
-print "iurt: checking basesystem tar\n";
 
 my $debug_tag = '_debug' if $debug;
 my $chroot = "$config->{local_home}/chroot$debug_tag";
 my $chroot_tar = "$chroot-$distro_tag.$my_arch.tar.gz";
-perform_command("sudo $config->{install_chroot_binary} cooker $config->{basesystem_media} $chroot_tar $chroot 501 basesystem rpm-build", 
-        mail => $config->{admin},
+if (!$nocheckchroot) {
+    print "iurt: checking basesystem tar\n";
+    system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot"});
+    perform_command("sudo $config->{install_chroot_binary} cooker $config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build", 
+	mail => $config->{admin},
 	error => "[REBUILD] Creating the inital chroot for $distro_tag on $my_arch failed", 
 	hash => 'chroot_inititialization', 
+	timeout => 600, 
 	debug_mail => $debug,
 	die => 1);
 
+    if ($unionfs) {
+	clean_chroot($chroot) or die "FATAL iurt: Could no prepare initial chroot"
+    }
+}
+
 my $local_spool = "$config->{local_home}/iurt/$distro_tag/$my_arch";
 if (!-d "$config->{local_home}/iurt/$distro_tag/") {
-	mkdir "$config->{local_home}/iurt/$distro_tag";
-	if (!-d $local_spool) {
-		mkdir $local_spool;
-		mkdir "$local_spool/log"
-	}
+    mkdir "$config->{local_home}/iurt/$distro_tag";
+    if (!-d $local_spool) {
+	mkdir $local_spool;
+	mkdir "$local_spool/log"
+    }
 }
 
 my %done;
-foreach my $t (@todo) {
-	my ($dir, $srpm) = @$t;
+my $wait_limit;
+my $done;
+my $home = $config->{local_home};
+foreach (my $i ; $i < @todo; $i++) {
+	my ($dir, $srpm) = @{$todo[$i]};
 	$done{$srpm} and next;
 	$done{$srpm} = 1;
 	check_version($srpm) or next;
 	if ($debug) { $debug++ == 2 and exit }
-	print "iurt: installing a new chroot for $srpm in $chroot\n";
-	-d $chroot and perform_command("sudo rm -rf $chroot", 
-		mail => $config->{admin},
-		error => "[REBUILD] Deleting of old chroot $chroot failed", 
-		hash => 'chroot_deletion', 
-		debug_mail => $debug,
-		die => 1);
-	mkdir $chroot;
-	perform_command("pushd $chroot && sudo tar xvf $chroot_tar", 
-		mail => $config->{admin}, 
-		error => "[REBUILD] creating the initial chroot $chroot failed", 
-		hash => 'chroot_init', 
-		debug_mail => $debug,
-		die => 1);
-
-	dump_rpmmacros("$chroot/home/builder/.rpmmacros") or next;
-
+	$done++;
+	# dump cache every ten packages (we can be killed by other iurt if we are stuck)
+	if (!$done % 10) { dump_cache() }
+	print "iurt: packages $srpm [$done/$to_compile]\n";
+	if ($unionfs) {
+	    foreach my $t ("unionfs",'tmpfs') {
+		my $d = "$home/$t";
+		while (check_mounted($d, $t)) {
+		    print STDERR "iurt: umounting $d\n";
+		    system(qq{sudo umount $d}) and die "FATAL iurt: could not umount $d"
+		}
+		print STDERR "iurt: removing $d\n";
+		system(qq{sudo rm -rf $d});
+		mkdir "$d";
+	    }
+	    system(qq{sudo mount -t tmpfs none $home/tmpfs}) and die "FATAL iurt: could not mount $home/tmpfs ($!)";
+	    system(qq{sudo mount -o dirs=$home/tmpfs=rw:$home/chroot/=ro -t unionfs unionfs $home/unionfs}) and die "FATAL iurt: could not mount $home/tmpfs and $home/chroot with unionfs ($!)";
+	} else {
+	    print "iurt: installing a new chroot for $srpm in $chroot\n";
+	    clean_chroot($chroot)
+	}
 	my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
 	my $maintainer = `rpmmon -s -p $srpm_name`;
 	my $cc = "$maint{$srpm}, maintainers\@mandriva.com";
@@ -273,30 +317,35 @@ foreach my $t (@todo) {
 	#($maintainer, $cc) = ($config->{admin},'');
 	print "Installing build dependencies of $srpm...\n";
 	# FIXME unfortunately urpmi stalls quite often
-	system(qq{sudo pkill -9 -u root -f "urpmi --root $chroot"});
-	perform_command("sudo urpmi -v --root $chroot --no-verify-rpm -s --auto $dir/$srpm", 
+	system(qq{sudo pkill -9 -u root -f "$todo[$i-1][1]"}) if $i - 1 >= 0;
+	system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --rrot $chroot"});
+	perform_command("sudo urpmi $urpmi_options --root $chroot $dir/$srpm", 
 		mail => $maintainer, 
 		error => "[REBUILD] install of build dependencies of $srpm failed on $my_arch",  
 		hash => "install_deps_$srpm", 
 		timeout => 600, 
-		freq => 48,
+		freq => 1,
 		cc => $cc, 
 		debug_mail => $debug,
 		error_regexp => 'cannot be installed',
 		wait_regexp => 'database locked',
+		wait_callback => sub { $wait_limit++; if ($wait_limit > 10) { $wait_limit = 0; system(qq{sudo pkill -9 urpmi}) } },
 		log => "$local_spool/log/",
 		callback => sub { 
 			my ($opt, $output) = @_;
 			print "Calling callback for $opt->{hash}\n" if $debug;
-			my ($missing_deps) = $output =~ /\(due to unsatisfied ([^[)]*)/;
+			my ($missing_deps, $version) = $output =~ /\(due to unsatisfied ([^[)]*)(\[.*\])?/;
 			$missing_deps or return;
 			my $other_maint = `rpmmon -p $missing_deps`;
+			$version ||= 1;
+			# remember what is needed, and do not try to recompile until it is available
 			chomp $other_maint;
 			print "Missing Dep: $missing_deps ($other_maint)\n";
 			if ($other_maint) {
 			    $opt->{mail} = $other_maint;
 			    $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $my_arch";
 			}
+			push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
 		},
 	) or next;
 	perform_command("sudo chroot $chroot rpm -qa", 
@@ -322,10 +371,12 @@ foreach my $t (@todo) {
 			log => "$local_spool/log/", 
 			error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
 			freq => 1) && !glob "$chroot/home/builder/rpm/RPMS/*/*.rpm") {
-		$cache->{failure}{$srpm}{$my_arch} = 1;
+		$cache->{failure}{$srpm} = 1;
 		next
 	}
-	if (!perform_command("sudo urpmi --root $chroot --no-verify-rpm --auto $chroot/home/builder/rpm/RPMS/*/*.rpm", 
+	# do some cleaning if the compilation is successful
+	delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm};
+	if (!perform_command("sudo urpmi $urpmi_options --root $chroot $chroot/home/builder/rpm/RPMS/*/*.rpm", 
 			mail => $maintainer, 
 			error => "[REBUILD] binaries packages generated from $srpm do not install correctly", 
 			hash => "binary_test_$srpm", 
@@ -334,7 +385,7 @@ foreach my $t (@todo) {
 			freq => 1, 
 			error_regexp => 'unable to access',
 			log => "$local_spool/log/")) {
-		$cache->{failure}{$srpm}{$my_arch} = 1;
+		$cache->{failure}{$srpm} = 1;
 		next
 	}
 	if ($debug) {
@@ -344,6 +395,7 @@ foreach my $t (@todo) {
 	    system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_spool") and print "ERROR: could not copy rpm files from $chroot/home/builder/rpm/RPMS/ to $local_spool ($!)\n";
 	    process_queue()
 	}
+	end:
 }
 
 print "iurt: reprocess generated packages queue\n";
@@ -385,20 +437,75 @@ sub usage {
 	exit
 }
 
+sub clean_chroot {
+    my ($chroot) = @_;
+    -d $chroot and perform_command("sudo rm -rf $chroot", 
+		mail => $config->{admin},
+		error => "[REBUILD] Deleting of old chroot $chroot failed", 
+		hash => 'chroot_deletion', 
+		debug_mail => $debug,
+		die => 1);
+   mkdir $chroot;
+   perform_command("pushd $chroot && sudo tar xvf $chroot_tar", 
+		mail => $config->{admin}, 
+		error => "[REBUILD] creating the initial chroot $chroot failed", 
+		hash => 'chroot_init', 
+		debug_mail => $debug,
+		die => 1);
+
+   dump_rpmmacros("$chroot/home/builder/.rpmmacros") or return;
+   1
+}
+
+sub check_mounted {
+    my ($mount_point, $type) = @_;
+    open my $mount, '/proc/mounts' or die 'FATAL iurt: could not open /proc/mounts';
+    $mount_point =~ s,//+,/,g;
+    while (<$mount>) {
+	return 1 if /^\w+ $mount_point $type /
+    }
+    0
+}
+
+sub check_needed {
+    my ($srpm) = @_;
+    if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 } 
+    my @n;
+    my $ok = 1;
+    foreach my $t (@{$cache->{needed}{$srpm}}) {
+	my ($name, $version, $maint) = @$t;
+	my ($p_version) = $provides{$name};
+	if ($p_version) {
+	    next if $version == $p_version;
+	    next if URPM::ranges_overlap($version, $p_version)
+	}
+	$ok = 0;
+	push @n, [ $name, $version ];
+	# try to recompile it once in a while
+	return 1 if ! $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72
+    }
+    delete $cache->{needed}{$srpm};
+    $cache->{needed}{$srpm} = \@n if @n;
+    $ok
+}
+
 sub process_queue {
 	my $dir = "$config->{local_home}/iurt/$distro_tag/$my_arch";
 	opendir my $rpmdir, $dir or next;
 	foreach my $rpm (readdir $rpmdir) {
 		my ($rarch, $srpm) = update_srpm($dir, $rpm);
 		$rarch or next;
-		# try to keep the opportunity to prevent disk full
-		my $ok = copy "$dir/$rpm", "$config->{upload}/RPMS/";
-		if (!$ok){
+		# recheck if the package has not been uploaded in the meantime
+		if (! -f "$rpms_dir/$rpm") {
+		    my $ok = copy "$dir/$rpm", "$config->{upload}/RPMS/";
+		    # try to keep the opportunity to prevent disk full
+		    if (!$ok){
 			print "ERROR process_queue: cannot copy $dir/$rpm to $config->{upload}/RPMS/ ($!)\n";
 			next
+		    }
 		}
 		unlink "$dir/$rpm";
-		$cache->{queue}{$srpm}{$rarch} = 1
+		$cache->{queue}{$srpm} = 1
 	}
 	closedir $rpmdir;
 }
@@ -508,22 +615,29 @@ sub perform_command {
 	} else {
 	    $output = "Command failed: $command\n$output"
 	}
-	if ($opt{callback}) {
+	if (ref $opt{callback}) {
 	    $opt{callback}(\%opt, $output)
 	}
-	if ($output =~ /$opt{wait_regexp}/) {
+	if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/) {
+	    $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
+	    print STDERR "ERROR iurt: $opt{wait_regexp} !\n";
+	    sendmail($config->{admin}, '' , "$opt{hash} on $my_arch for $media: could not proceed", "$opt{wait_regexp}\n\n$output", 0, 0, $opt{debug_mail});
+	    if ($opt{die}) {
+		dump_cache();
+		die "FATAL iurt: $opt{error}."
+	    }
 	    return 0
 	}
 	if ($? || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) {
 	    if ($opt{mail} && $config->{sendmail}) {
-		if (! $cache->{warning}{$opt{hash}}{$my_arch}{$opt{mail}} % $opt{freq}) {
+		if (! $cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq}) {
 		    sendmail($opt{mail}, $opt{cc} , $opt{error} , $output, 0, 0, $opt{debug_mail});
 		} elsif ($config->{admin}) {
 		    sendmail($config->{admin}, '' , $opt{error}, $output, 0, 0, $opt{debug_mail});
 		}
 	    }
-	    $cache->{warning}{$opt{hash}}{$my_arch}{$opt{mail}}++;
-	    print "\n$output\n";
+	    $cache->{warning}{$opt{hash}}{$opt{mail}}++;
+	    print STDERR "\n$output\n";
 	    if ($opt{die}) {
 		dump_cache();
 		die "FATAL iurt: $opt{error}."