switch to perl-RPM4; try to guess missing buildrequires; add find_provides; add missing_provides to report

1 files changed, 79 insertions, 29 deletions

diff --git a/iurt2 b/iurt2
index 9f78e2e..bdfc195 100755
--- a/iurt2
+++ b/iurt2
@@ -30,7 +30,7 @@
 # - Maybe call the function from the initial todo list (thus making the argument ordering important)
 #
 use strict;
-use Hdlist;
+use RPM4::Header;
 use Data::Dumper;
 use URPM;
 use File::NCopy qw(copy);
@@ -38,6 +38,7 @@ use MIME::Words qw(encode_mimewords);
 # I did not manage to make locks work over the network
 #use File::lockf;
 use Mkcd::Commandline qw(parseCommandLine usage);
+use MDK::Common;
 use Filesys::Df qw(df);
 
 my $program_name = 'iurt2';
@@ -139,7 +140,7 @@ $run{todo} = [ ];
 	    } else {
 		die "FATAL iurt: $_ does not seems to be a SRPM\n"
 	    }
-	    my $hdr = rpm2header($_);
+	    my $hdr = RPM4::Header->new($_);
 	    if (check_arch($hdr)) {
 		print {$run{LOG}} "iurt: force build for $2 (from $1)\n";
 		push @{$run{todo}}, [ $path, $srpm, 1 ]
@@ -384,12 +385,14 @@ if ($run{unionfs}) {
     if (!$ok) {
 	print {$run{LOG}} "iurt: adding unionfs module\n" if $run{verbose} > 0;
 	system("sudo /sbin/depmod -a");
-	system("sudo /sbin/modprobe -f unionfs");
+	system("sudo /sbin/modprobe -f unionfs") and $run{unionfs} = 0;
+    }
+    if ($run{unionfs}) {
+	$unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag/";
+	remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs);
+	$unionfs_dir = "$unionfs_dir/$run{user}/";
+	-d $unionfs_dir or mkdir $unionfs_dir
     }
-    $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag/";
-    remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs);
-    $unionfs_dir = "$unionfs_dir/$run{user}/";
-    -d $unionfs_dir or mkdir $unionfs_dir
 }
 
 my (%done, $wait_limit, $done);
@@ -437,23 +440,17 @@ if ($run{shell}) {
 }
 
 $config->{local_upload} ||= $config->{local_home};
-my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}";
+my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/";
 if (!-d $local_spool) {
     print {$run{LOG}} "iurt: creating local spool $local_spool\n" if $run{verbose} > 4;
-    -d "$config->{local_upload}/iurt" or mkdir "$config->{local_upload}/iurt";
-    my $d = "$config->{local_upload}/iurt/$run{distro_tag}";
-    if (!-d $d) { mkdir $d or die "FATAL iurt: could not create local spool dir $d ($!)" }
-    if (!-d $local_spool) {
-	mkdir $local_spool;
-	mkdir "$local_spool/log"
-    }
+    mkdir_p("$local_spool/log") or die "FATAL iurt: could not create local spool dir $local_spool ($!)"
 }
 
 # perform some cleaning before running to have some more space, rsync to the server too in case previous iurt crashed
 if ($config->{rsync_to} && !$run{no_rsync}) { 
     	# remove some old and very big log files not to saturate the server
 	system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|);
-	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/");
+	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/");
 }
 
 print {$run{LOG}} "iurt: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3;
@@ -620,12 +617,12 @@ do {
 			    print {$run{LOG}} "missing dep: $missing_deps ($other_maint) missing_package $missing_package ($first_maint)\n";
 			    foreach my $m ($first_maint, $other_maint) {
 				if ($other_maint && $other_maint ne 'NOT_FOUND') {
-				    $opt->{mail} = "warly\@mandriva.com";
+				    $opt->{mail} = $config->{admin};
 				    #$opt->{mail} .= ", $other_maint";
 				}
 			    }
 			    if (!$opt->{mail}) { 
-				$opt->{mail} = "warly\@mandriva.com";
+				$opt->{mail} = $config->{admin}
 			    }
 			    # remember what is needed, and do not try to recompile until it is available
 			    if ($missing_package) {
@@ -633,7 +630,7 @@ do {
 				$cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $first_maint || $other_maint || $maintainer };
 			    } else {
 				$opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
-				$cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $other_maint || $maintainer };
+				$cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $maintainer || $other_maint };
 			    }
 			} 
 			0
@@ -678,7 +675,7 @@ do {
 			system("sudo chroot $chroot_tmp /bin/su $luser -c bash");
 			exit
 		    }
-		    print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug};
+		    print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{verbose} > 4;
 		    if ($unionfs_tmp && $output =~ /no space left on device/i) {
 			print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
 			$cache->{no_unionfs}{$srpm} = 1;
@@ -687,6 +684,26 @@ do {
 			print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n";
 			$cache->{no_unionfs}{$srpm} = 1;
 			return 1
+		    } elsif ($output =~ /configure.*error.* -l([^\s]+)/) {
+			my $missing = $1;
+			my @rpm = find_provides(\%run, \%pack_provide, $missing);
+			print {$run{LOG}} "iurt: likely @rpm ($missing-devel) needed to rebuilt $srpm is not in build_requires\n" if $run{verbose} > 4;
+			if ($maintainer ne 'NOT_FOUND') {
+			    $opt->{mail} = $config->{admin};
+			    #$opt->{mail} .= ", $other_maint";
+			}
+			if (!$opt->{mail}) { 
+			    $opt->{mail} = $config->{admin};
+			}
+			if (@rpm > 1) {
+			    $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] one of @rpm ($missing-devel), needed to build $srpm, is not in buildrequires";
+			} elsif (@rpm == 1) {
+			    $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] @rpm ($missing-devel), needed to build $srpm, is not in buildrequires";
+			} else {
+			    $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] $missing-devel, needed to build $srpm, is not in buildrequires";
+			}
+			$cache->{buildrequires}{$srpm}{$missing} = \@rpm;
+			return 
 		    }
 		    1
 		}, 
@@ -708,8 +725,8 @@ do {
 		    goto retry
 		}
 		$cache->{failure}{$srpm} = 1;
-		# 20060615
 		$run{status}{$srpm} = 'build_failure';
+		# 20060615
 		dump_cache(\%run);
 		dump_status($local_spool, \%run);
 		next
@@ -717,6 +734,7 @@ do {
 	}
 	# do some cleaning if the compilation is successful
 	delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm};
+	delete $cache->{buildrequires}{$srpm} if defined $cache->{buildrequires}{$srpm};
 	if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", 
 		\%run, $config,
 		#	mail => $maintainer, 
@@ -734,7 +752,7 @@ do {
 	    next
 	}
 	$run{status}{$srpm} = 'ok';
-	delete $cache->{failure}{$srpm};
+	delete $cache->{failure}{$srpm} if defined $cache->{failure}{$srpm};
 	if ($run{debug}) {
 	    print {$run{LOG}} "iurt: debug mode, skip other packages\n";
 	    exit
@@ -779,7 +797,7 @@ send_status_mail(\%run, $config, $cache) if ($run{status_mail});
 if ($config->{rsync_to} && !$run{no_rsync}) { 
     	# remove some old and very big log files not to saturate the server
 	system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|);
-	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/");
+	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/");
 }
 
 # one last try to clean
@@ -965,6 +983,7 @@ sub check_needed {
     my $ent = $cache->{needed}{$srpm};
     if (ref $ent eq 'ARRAY') {
 	my $table = $ent;
+	$cache->{needed}{$srpm} = {};
 	foreach my $t (@$table) {
 	    my ($missing, $version, $maint) = @$t;
 	    $cache->{needed}{$srpm}{$missing} = {
@@ -972,6 +991,7 @@ sub check_needed {
 		maint => $maint
 	    }
 	}
+	$ent = $cache->{needed}{$srpm}
     }
     foreach my $name (keys %$ent) {
 	my ($package, $version, $maint) = @{$ent->{$name}}{'package', 'version','maint'};
@@ -1003,7 +1023,7 @@ sub check_needed {
 sub process_queue {
     my ($config, $run, $wrong_rpm, $quiet) = @_;
     return if !$run->{upload} && $quiet;
-    my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}";
+    my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}/$run->{media}/";
     opendir my $rpmdir, $dir or return;
     foreach my $rpm (readdir $rpmdir) {
 	my ($rarch, $srpm) = update_srpm($dir, $rpm, $wrong_rpm);
@@ -1034,7 +1054,7 @@ sub update_srpm {
 	my ($arch) = $rpm =~ /([^\.]+)\.rpm$/ or return 0;
 	my $srpm = $cache->{rpm_srpm}{$rpm};
 	if (!$srpm) {
-		my $hdr = rpm2header("$dir/$rpm");
+		my $hdr = RPM4::Header->new("$dir/$rpm");
 		$hdr or return 0;
 		$srpm = $hdr->queryformat("%{SOURCERPM}");
 		$cache->{rpm_srpm}{$rpm} = $srpm
@@ -1058,7 +1078,7 @@ sub dump_cache {
 	if (-f $filename) {
 	    print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
 	    $old_cache = do $cachefile;
-	    foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning') {
+	    foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning', 'buildrequires') {
 		foreach my $rpm (%{$old_cache->{$k}}) {
 		    $cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm}
 		}
@@ -1197,7 +1217,7 @@ sub perform_command {
 	    print {$run{LOG}} "$command\n" if $run{verbose} > 2;
 	    if ($opt{log}) {
 		#$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
-		system("$command 2>&1 >> $logfile");
+		system("$command &> $logfile");
 	    } else {
 		$output = `$command 2>&1`;
 	    }
@@ -1506,7 +1526,7 @@ sub search_packages {
 		my $check_needed = check_needed($srpm, $cache, $provides);
 		$run->{status}{$srpm} = 'missing_buildrequires' if !$check_needed;
 		if (!$cache->{queue}{$srpm} && $check_needed) {
-		    my $hdr = rpm2header("$dir/$srpm");
+		    my $hdr = RPM4::Header->new("$dir/$srpm");
 		    if (!check_arch($hdr)) {
 			$run->{status}{$srpm} = 'not_on_this_arch';
 			next
@@ -1874,12 +1894,31 @@ sub send_status_mail {
 	    }
         } elsif ($run->{status}{$rpm} eq 'build_failure') {
 	    my ($maint) = get_maint($run, $rpm);
-	    push @{$output{build}{$maint}}, $rpm
+	    if ($cache->{buildrequires}{$rpm}) {
+		push @{$output{buildrequires}{$maint}}, $rpm
+	    } else {
+		push @{$output{build}{$maint}}, $rpm
+	    }
 	} elsif (!$run->{status}{$rpm}) {
 	    # need to find something more usefull to do at that point
 	    next
 	}
     }
+    my $text = "*** Missing buildrequires tag in specfile ***\n";
+    foreach my $maint (keys %{$output{buildrequires}}) {
+	$text .= "\n$maint\n";
+	foreach my $pack (keys %{$output{missing}{$maint}}) {
+	    foreach my $missing (keys %{$cache->{buildrequires}{$pack}}) {
+		my $rpms = $cache->{buildrequires}{$pack}{$missing};
+		if (@$rpms) {
+		    $text .= "  $pack should have a buildrequires on @$rpms (for $missing-devel)\n"
+		} else {
+		    $text .= "  $pack should have a buildrequires for $missing-devel\n"
+		}
+	    }
+	}
+    }
+
     my $text = "*** Missing dependencies ***\n";
     foreach my $maint (keys %{$output{missing}}) {
 	$text .= "\n$maint\n";
@@ -1920,3 +1959,14 @@ sub get_maint {
     $run{maint}{$srpm} = $maint;
     $maint, $srpm_name
 }
+
+sub find_provides {
+    my ($run, $pack_provide, $p) = @_;
+    my @rpm;
+    foreach my $provides (keys %{pack_provide}) {
+	if ($provides =~ /$p/ && $provides =~ /devel/) {
+	    push @rpm, $pack_provide->{$provides};
+	}
+    }
+    @rpm
+}