- add a --config_help option to display available configuration file parameters

- correctly clean chroot when switching from a non unionfs to a unionfs build
- try to catch the perl problem in unionfs where /proc/self/exe is not correct

1 files changed, 115 insertions, 60 deletions

diff --git a/iurt2 b/iurt2
index 7eefc12..3e81a1b 100755
--- a/iurt2
+++ b/iurt2
@@ -24,8 +24,6 @@
 # TODO
 #
 # - use a cache (rpmctl cache for example) to find maintainer
-# - Do not run if the hdlist are broken
-# - allow to force packages to rebuild iurt foo-2.3-1mdk.src.rpm
 #
 use strict;
 use Hdlist;
@@ -36,7 +34,7 @@ use MIME::Words qw(encode_mimewords);
 use Fcntl ':flock';
 
 # maybe time to switch to mkcd::parse_command_line
-my ($debug, $unionfs, $nocheckchroot, $force_build, @todo);
+my ($debug, $unionfs, $nocheckchroot, $force_build, @todo, $config_usage);
 my @argv = grep { 
     if ($force_build && !/--/) {
 	if (m,(.*/)([^/]*.src.rpm)$, && -f $_) {
@@ -52,6 +50,8 @@ my @argv = grep {
 	elsif (/--unionfs/) { $unionfs = 1; 0 }
 	elsif (/--nocheckchroot/) { $nocheckchroot = 1; 0 }
 	elsif (/--force_build/) { $force_build = 1; 0 }
+	elsif (/--help/) { usage(); 0 }
+	elsif (/--config_help/) { $config_usage = 1; 0 }
 	elsif (/--./) { usage("$_ is an unknown option") }
 	else { 1 }
     }
@@ -65,7 +65,6 @@ my $media = shift @argv;
 my @special_srpm_dir = @argv;
 my $urpmi_options = "-v --no-verify-rpm --nolock --auto";
 
-$my_arch or usage("no architecture given (media $media, my_arch $my_arch, todo @todo)");
 my $real_arch = `uname -m`;
 chomp $real_arch;
 my %arch_comp = (
@@ -75,9 +74,6 @@ my %arch_comp = (
     'ppc' => { 'ppc' => 1 },
     'ppc64' => { 'ppc' => 1, 'ppc64' => 1 },
 );
-if (!$arch_comp{$real_arch}{$my_arch}) {
-    die "FATAL iurt: could not compile $my_arch binaries on a $real_arch"
-}
 
 my $HOME = $ENV{HOME};
 my $configfile = "$HOME/.iurt.$distro_tag.conf";
@@ -89,21 +85,32 @@ if (-f $configfile) {
     $config = {}
 }
 
-$config->{home} ||= $HOME;
-$config->{cache_home} ||= "$HOME/.bugs";
-$config->{supported_arch} ||= ['i586', 'x86_64'];
-$config->{upload} ||= "$HOME/uploads/";
-$config->{local_home} ||= "$HOME";
-$config->{repository} ||= "/mnt/BIG/dis/";
-$config->{packager} ||= "Iurt";
-$config->{install_chroot_binary} ||= 'install-chroot-tar.sh';
-$config->{distribution} ||= 'Mandriva Linux';
-$config->{vendor} ||= 'Mandriva';
-$config->{basesystem_media} ||= "$config->{repository}/$distro_version/$my_arch/media/main/";
-$config->{all_media} ||= [ 'main', 'contrib' ];
-$config->{log_size_limit} ||= '100M';
-$config->{log_size_date} ||= '30';
-$config->{no_mail} ||= { };
+my %config_usage = ( 
+    home => { desc => 'Home dir', default => $HOME },
+    cache_home => { desc => 'Where to store the cache files', default => "$HOME/.bugs" },
+    supported_arch => { desc => 'Table of supported architecture', default => ['i586', 'x86_64'] },
+    upload => { desc => 'Where to copy build packages', default => "$HOME/uploads/" },
+    local_home => { desc => 'Where to build packages', default => $HOME },
+    repository => { desc => 'Prefix of the repositories', default => '/mnt/BIG/dis/' },
+    packager => { desc => 'Name of the build bot', default => 'Iurt' },
+    install_chroot_binary => { desc => 'Tool used to create initial chroot', default => 'install-chroot-tar.sh' },
+    distribution => { desc => 'Name of the packages distribution', default => 'Mandriva Linux' },
+    vendor => { desc => 'Name of the packages vendor', default => 'Mandriva' },
+    basesystem_media => { desc => 'Where to find basesystem packages', default => "$config->{repository}/$distro_version/$my_arch/media/main/" },
+    all_media => { desc => 'List of known media', default => [ 'main', 'contrib' ] },
+    log_size_limit => { desc => 'Maximum authorized size for a log file', default => '100M' },
+    log_size_date => { desc => 'Number of days log should be kept', default => '30' },
+    no_mail  => { desc => 'Hash table with people mail address where we should not send any mails', default => {} },
+    minimum_package_number => { "Minimum number of packages in a synthesis file to consider it valid", default => 1000 },
+   );
+config_usage() if $config_usage;
+$my_arch or usage("no architecture given (media $media, my_arch $my_arch, todo @todo)");
+if (!$arch_comp{$real_arch}{$my_arch}) {
+    die "FATAL iurt: could not compile $my_arch binaries on a $real_arch"
+}
+foreach my $k (keys %config_usage) {
+    $config->{$k} ||= $config_usage{$k}{default}
+}
 
 $config->{upload} .= $distro_version;
 $config->{upload} =~ s/community//g;
@@ -191,6 +198,7 @@ foreach my $m (@{$config->{all_media}}) {
 	print STDERR "Parsing $synthesis_file\n";
 	if (open my $syn, "zcat $synthesis_file |") { 
 	    my @prov;
+	    my $nb;
 	    while (<$syn>) {
 		if (/^\@provides@(.*)/) {
 		    foreach my $p (split '@', $1) {
@@ -199,15 +207,17 @@ foreach my $m (@{$config->{all_media}}) {
 			$provides{$1} = $2 || 1
 		    }
 		} elsif (/\@info\@([^@]+)@/) {
+		    $nb++;
 		    my ($name) = $1 =~ /(.*)-[^-]+-[^-]+\..*$/;
 		    foreach (@prov) {
 			$pack_provide{$_} = $name
 		    }
 		    @prov = ()
 		}
-	    }	    
+	    }
+	    $nb < $config->{minimum_package_number} and die "FATAL iurt: synthesis files seems corrupted, only $nb packages found."
 	} else {
-	    print STDERR "ERROR: Could not open $synthesis_file, will not precheck dependecies avaibility\n";
+	    die "FATAL iurt: Could not open $synthesis_file\n";
 	}
     }
 }
@@ -333,14 +343,7 @@ foreach (my $i ; $i < @todo; $i++) {
 	    dump_cache();
 	    die "FATAL iurt: Could not have urpmi working !"
 	}
-	my $need_chroot_cleaning;
-	if ($cache->{no_unionfs}{$srpm}) {
-	    $unionfs_tmp = 0
-	} elsif ($unionfs) {
-	    $need_chroot_cleaning = 1 if !$unionfs_tmp;
-	    $unionfs_tmp = 1
-	}
-	if ($unionfs_tmp && !$need_chroot_cleaning) {
+	if ($unionfs_tmp) {
 	    my $mount_point = "$unionfs_dir/unionfs.$run.$union_id";
 	    print STDERR "Cleaning $mount_point\n";
 	    if (!clean_mnt($mount_point)) {
@@ -355,9 +358,17 @@ foreach (my $i ; $i < @todo; $i++) {
 	    $chroot = "$unionfs_dir/unionfs.$run.$union_id";
 	    mkdir $tmpfs or die "Could not create $tmpfs ($!)";
 	    mkdir $chroot or die "Could not create $chroot ($!)";
-	    system(qq{sudo mount -t tmpfs none $tmpfs}) and die "FATAL iurt: could not mount $tmpfs ($!)";
-	    system(qq{sudo mount -o dirs=$tmpfs=rw:$home/chroot=ro -t unionfs none $chroot}) and die "FATAL iurt: could not mount $tmpfs and $home/chroot with unionfs ($!)";
-	    system("sudo mount none -t proc $chroot/proc") and die "FATAL iurt: could not mount /proc in the chroot.";
+	    if ($cache->{no_unionfs}{$srpm}) {
+		$unionfs_tmp = 0;
+		clean_chroot($chroot)
+	    } else {
+		# if the previous package has been built without unionfs, chroot need to be cleaned
+		clean_chroot($chroot) if !$unionfs_tmp;
+		$unionfs_tmp = 1;
+		system(qq{sudo mount -t tmpfs none $tmpfs}) and die "FATAL iurt: could not mount $tmpfs ($!)";
+		system(qq{sudo mount -o dirs=$tmpfs=rw:$home/chroot$debug_tag=ro -t unionfs none $chroot}) and die "FATAL iurt: could not mount $tmpfs and $home/chroot with unionfs ($!)";
+		system("sudo chroot $chroot mount none -t proc /proc") and die "FATAL iurt: could not mount /proc in the chroot $chroot.";
+	    }
 	} else {
 	    print STDERR "iurt: installing a new chroot for $srpm in $chroot\n";
 	    clean_chroot($chroot)
@@ -371,11 +382,29 @@ foreach (my $i ; $i < @todo; $i++) {
 		#$cc = 'maintainers@mandriva.com'
 	}
 	#($maintainer, $cc) = ($config->{admin},'');
+	
+	# recreate a new srpm for buildarch condition in the spec file
+	print "Copying $srpm to $chroot\n";
+	perform_command("sudo cp $dir/$srpm $chroot/home/builder/rpm/SRPMS/", 
+		mail => $config->{admin}, 
+		error => "[REBUILD] cannot copy $srpm to $chroot", 
+		debug_mail => $debug,
+		hash => "copy_$srpm") or next;
+	perform_command(qq{sudo chroot $chroot su builder -c "rpm -i /home/builder/rpm/SRPMS/$srpm"}, 
+		mail => $config->{admin}, 
+		error => "[REBUILD] cannot install $srpm in $chroot", 
+		debug_mail => $debug,
+		hash => "install_$srpm") or next;
+	perform_command(qq{sudo chroot $chroot su builder -c "rpm --nodeps -bs /home/builder/rpm/SPECS/*.spec"}, 
+		mail => $config->{admin}, 
+		error => "[REBUILD] cannot create $srpm in $chroot", 
+		debug_mail => $debug,
+		hash => "create_$srpm") or next;
 	print "Installing build dependencies of $srpm...\n";
 	
-	#system(qq{sudo pkill -9 dd-u root -f "$todo[$i-1][1]"}) if $i - 1 >= 0;
+	#system(qq{sudo pkill -9 -u root -f "$todo[$i-1][1]"}) if $i - 1 >= 0;
 	#system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot"});
-	perform_command("sudo urpmi $urpmi_options --root $chroot $dir/$srpm", 
+	perform_command("sudo urpmi $urpmi_options --root $chroot $chroot/home/builder/rpm/SRPMS/$srpm", 
 		mail => $maintainer, 
 		error => "[REBUILD] install of build dependencies of $srpm failed on $my_arch",  
 		hash => "install_deps_$srpm", 
@@ -410,14 +439,7 @@ foreach (my $i ; $i < @todo; $i++) {
 		timeout => 60, 
 		debug_mail => $debug,
 		log => "$local_spool/log/") or next;
-	print "Copying $srpm to $chroot\n";
-	perform_command("sudo cp $dir/$srpm $chroot/home/builder/rpm/SRPMS/", 
-		mail => $config->{admin}, 
-		error => "[REBUILD] cannot copy $srpm to $chroot", 
-		debug_mail => $debug,
-		hash => "copy_$srpm") or next;
 	print "Compiling $srpm\n";
-	#system(qq{sudo chroot $chroot /bin/su builder -c "mkdir rpm/RPMS/x86_64 rpm/RPMS/noarch"});
 	if (!perform_command(qq{TMP=/home/builder/tmp/ sudo chroot $chroot /bin/su builder -c "rpm --rebuild /home/builder/rpm/SRPMS/$srpm"}, 
 			mail => $maintainer, 
 			error => "[REBUILD] $srpm from $distro_tag does not build correctly on $my_arch", 
@@ -433,6 +455,9 @@ foreach (my $i ; $i < @todo; $i++) {
 			    if ($unionfs_tmp && $output =~ /no space left on device/i) {
 				print STDERR "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
 				$cache->{no_unionfs}{$srpm} = 1
+			    } elsif ($unionfs_tmp && $output =~ m,$home,) {
+				print STDERR "ERROR iurt: seems like building $srpm needs to acces /proc/self/exe, which is broken with unionfs, will recompile it in normal mode\n";
+				$cache->{no_unionfs}{$srpm} = 1
 			    }
 			}, 
 			freq => 1) && !glob "$chroot/home/builder/rpm/RPMS/*/*.rpm") {
@@ -518,18 +543,41 @@ sub usage {
 			e.g. iurt community/2006.0 x86_64 main
 
 		options:
+			--config_help: Display configuration file help
+			
 			--debug: Compile one package in debug mode (do not send mail, create
 				 chroot_debug directory). This mode can be used on a system
 				 where a iurt is already running.
+				 
 			--force_build <force_srpm_build_1> <force_srpm_build_2> ... <force_srpm_build_n>
 		       		      list of extra srpm to build
+				      
+			--help: Display this help
+			
+			--nocheckchroot: start building without the inital check of the chroot.
+			
 			--unionfs: Use a tmpfs mounted with the chroot via unionfs as build chroot.
 				   This is much faster but you need some memory.
-			--nocheckchroot: start building without the inital check of the chroot.
+
 ";
 	exit
 }
 
+sub config_usage {
+	print "
+	
+	Iurt configuration keywords:
+	
+";
+    $Data::Dumper::Indent = 0;
+    $Data::Dumper::Terse = 1;
+    foreach my $k (sort keys %config_usage) {
+	print "	$k: $config_usage{$k}{desc} (default: ", Data::Dumper->Dump([$config_usage{$k}{default}]), ", current: ", Data::Dumper->Dump([$config->{$k}]),")\n\n"
+    }
+    print "\n\n";
+    exit
+}
+
 sub clean_all_unionfs {
     my ($unionfs_dir) = @_;
     print STDERR "Cleaning old unionfs remaining dir in $unionfs_dir\n";
@@ -544,7 +592,7 @@ sub clean_all_unionfs {
 sub clean_chroot {
     my ($chroot, $only_clean) = @_;
     if (-d $chroot) {
-	system("sudo umount $chroot/proc");
+	system("sudo chroot $chroot umount /proc");
 	perform_command("sudo rm -rf $chroot", 
 	    mail => $config->{admin},
 	    error => "[REBUILD] Deleting of old chroot $chroot failed", 
@@ -562,7 +610,7 @@ sub clean_chroot {
 		die => 1);
 
    dump_rpmmacros("$chroot/home/builder/.rpmmacros") or return;
-   system("sudo mount none -t proc $chroot/proc") and return;
+   system("sudo chroot $chroot mount none -t proc /proc") and return;
    1
 }
 
@@ -592,22 +640,29 @@ sub clean {
 
 sub clean_unionfs {
     my ($unionfs_dir, $run, $union_id) = @_;
-    system("sudo umount $unionfs_dir/unionfs.$run.$union_id/proc") and print STDERR "ERROR iurt: could not umount $unionfs_dir/unionfs.$run.$union_id/proc\n";
+    system("sudo chroot $unionfs_dir/unionfs.$run.$union_id/ umount /proc") and print STDERR "ERROR iurt: could not umount /proc in $unionfs_dir/unionfs.$run.$union_id\n";
     print "Cleaning $unionfs_dir/unionfs.$run.$union_id\n";
-    foreach my $t ("unionfs",'tmpfs') {
-	# unfortunately quite oftem the unionfs is busy and could not be unmounted
-	my $d = "$unionfs_dir/$t.$run.$union_id";
-	while (check_mounted($d, $t)) {
-	    system("sudo fuser -k $d");
-	    print STDERR "iurt: umounting $d\n";
-	    if (system(qq{sudo umount $d &> /dev/null})) {
-		print STDERR "WARNING iurt: could not umount $d ($!)\n";
-		$union_id++;
-		return $union_id
+    my $nok = 1;
+    while ($nok) {
+	foreach my $t ("unionfs",'tmpfs') {
+	    # unfortunately quite oftem the unionfs is busy and could not be unmounted
+	    my $d = "$unionfs_dir/$t.$run.$union_id";
+	    my $last;
+	    while (check_mounted($d, $t)) {
+		system("sudo fuser -k $d");
+		print STDERR "iurt: umounting $d\n";
+		if (system(qq{sudo umount $d &> /dev/null})) {
+		    print STDERR "WARNING iurt: could not umount $d ($!)\n";
+		    $union_id++;
+		    $last = 1;
+		    last
+		}
 	    }
+	    $last and last;
+	    print STDERR "iurt: removing $d\n";
+	    system(qq{sudo rm -rf $d});
+	    $nok = 0
 	}
-	print STDERR "iurt: removing $d\n";
-	system(qq{sudo rm -rf $d});
     }
     $union_id
 }