From b488f31dd4cd5fc528621bddcb2369c41a7f8e63 Mon Sep 17 00:00:00 2001 From: Florent Villard Date: Wed, 21 Jun 2006 15:23:20 +0000 Subject: switch to perl-RPM4; try to guess missing buildrequires; add find_provides; add missing_provides to report --- iurt2 | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/iurt2 b/iurt2 index 9f78e2e..bdfc195 100755 --- a/iurt2 +++ b/iurt2 @@ -30,7 +30,7 @@ # - Maybe call the function from the initial todo list (thus making the argument ordering important) # use strict; -use Hdlist; +use RPM4::Header; use Data::Dumper; use URPM; use File::NCopy qw(copy); @@ -38,6 +38,7 @@ use MIME::Words qw(encode_mimewords); # I did not manage to make locks work over the network #use File::lockf; use Mkcd::Commandline qw(parseCommandLine usage); +use MDK::Common; use Filesys::Df qw(df); my $program_name = 'iurt2'; @@ -139,7 +140,7 @@ $run{todo} = [ ]; } else { die "FATAL iurt: $_ does not seems to be a SRPM\n" } - my $hdr = rpm2header($_); + my $hdr = RPM4::Header->new($_); if (check_arch($hdr)) { print {$run{LOG}} "iurt: force build for $2 (from $1)\n"; push @{$run{todo}}, [ $path, $srpm, 1 ] @@ -384,12 +385,14 @@ if ($run{unionfs}) { if (!$ok) { print {$run{LOG}} "iurt: adding unionfs module\n" if $run{verbose} > 0; system("sudo /sbin/depmod -a"); - system("sudo /sbin/modprobe -f unionfs"); + system("sudo /sbin/modprobe -f unionfs") and $run{unionfs} = 0; + } + if ($run{unionfs}) { + $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag/"; + remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs); + $unionfs_dir = "$unionfs_dir/$run{user}/"; + -d $unionfs_dir or mkdir $unionfs_dir } - $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag/"; - remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs); - $unionfs_dir = "$unionfs_dir/$run{user}/"; - -d $unionfs_dir or mkdir $unionfs_dir } my (%done, $wait_limit, $done); @@ -437,23 +440,17 @@ if ($run{shell}) { } $config->{local_upload} ||= $config->{local_home}; -my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}"; +my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/"; if (!-d $local_spool) { print {$run{LOG}} "iurt: creating local spool $local_spool\n" if $run{verbose} > 4; - -d "$config->{local_upload}/iurt" or mkdir "$config->{local_upload}/iurt"; - my $d = "$config->{local_upload}/iurt/$run{distro_tag}"; - if (!-d $d) { mkdir $d or die "FATAL iurt: could not create local spool dir $d ($!)" } - if (!-d $local_spool) { - mkdir $local_spool; - mkdir "$local_spool/log" - } + mkdir_p("$local_spool/log") or die "FATAL iurt: could not create local spool dir $local_spool ($!)" } # perform some cleaning before running to have some more space, rsync to the server too in case previous iurt crashed if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|); - system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/"); + system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/"); } print {$run{LOG}} "iurt: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3; @@ -620,12 +617,12 @@ do { print {$run{LOG}} "missing dep: $missing_deps ($other_maint) missing_package $missing_package ($first_maint)\n"; foreach my $m ($first_maint, $other_maint) { if ($other_maint && $other_maint ne 'NOT_FOUND') { - $opt->{mail} = "warly\@mandriva.com"; + $opt->{mail} = $config->{admin}; #$opt->{mail} .= ", $other_maint"; } } if (!$opt->{mail}) { - $opt->{mail} = "warly\@mandriva.com"; + $opt->{mail} = $config->{admin} } # remember what is needed, and do not try to recompile until it is available if ($missing_package) { @@ -633,7 +630,7 @@ do { $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $first_maint || $other_maint || $maintainer }; } else { $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}"; - $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $other_maint || $maintainer }; + $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $maintainer || $other_maint }; } } 0 @@ -678,7 +675,7 @@ do { system("sudo chroot $chroot_tmp /bin/su $luser -c bash"); exit } - print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug}; + print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{verbose} > 4; if ($unionfs_tmp && $output =~ /no space left on device/i) { print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n"; $cache->{no_unionfs}{$srpm} = 1; @@ -687,6 +684,26 @@ do { print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n"; $cache->{no_unionfs}{$srpm} = 1; return 1 + } elsif ($output =~ /configure.*error.* -l([^\s]+)/) { + my $missing = $1; + my @rpm = find_provides(\%run, \%pack_provide, $missing); + print {$run{LOG}} "iurt: likely @rpm ($missing-devel) needed to rebuilt $srpm is not in build_requires\n" if $run{verbose} > 4; + if ($maintainer ne 'NOT_FOUND') { + $opt->{mail} = $config->{admin}; + #$opt->{mail} .= ", $other_maint"; + } + if (!$opt->{mail}) { + $opt->{mail} = $config->{admin}; + } + if (@rpm > 1) { + $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] one of @rpm ($missing-devel), needed to build $srpm, is not in buildrequires"; + } elsif (@rpm == 1) { + $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] @rpm ($missing-devel), needed to build $srpm, is not in buildrequires"; + } else { + $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] $missing-devel, needed to build $srpm, is not in buildrequires"; + } + $cache->{buildrequires}{$srpm}{$missing} = \@rpm; + return } 1 }, @@ -708,8 +725,8 @@ do { goto retry } $cache->{failure}{$srpm} = 1; - # 20060615 $run{status}{$srpm} = 'build_failure'; + # 20060615 dump_cache(\%run); dump_status($local_spool, \%run); next @@ -717,6 +734,7 @@ do { } # do some cleaning if the compilation is successful delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm}; + delete $cache->{buildrequires}{$srpm} if defined $cache->{buildrequires}{$srpm}; if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", \%run, $config, # mail => $maintainer, @@ -734,7 +752,7 @@ do { next } $run{status}{$srpm} = 'ok'; - delete $cache->{failure}{$srpm}; + delete $cache->{failure}{$srpm} if defined $cache->{failure}{$srpm}; if ($run{debug}) { print {$run{LOG}} "iurt: debug mode, skip other packages\n"; exit @@ -779,7 +797,7 @@ send_status_mail(\%run, $config, $cache) if ($run{status_mail}); if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|); - system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/"); + system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/"); } # one last try to clean @@ -965,6 +983,7 @@ sub check_needed { my $ent = $cache->{needed}{$srpm}; if (ref $ent eq 'ARRAY') { my $table = $ent; + $cache->{needed}{$srpm} = {}; foreach my $t (@$table) { my ($missing, $version, $maint) = @$t; $cache->{needed}{$srpm}{$missing} = { @@ -972,6 +991,7 @@ sub check_needed { maint => $maint } } + $ent = $cache->{needed}{$srpm} } foreach my $name (keys %$ent) { my ($package, $version, $maint) = @{$ent->{$name}}{'package', 'version','maint'}; @@ -1003,7 +1023,7 @@ sub check_needed { sub process_queue { my ($config, $run, $wrong_rpm, $quiet) = @_; return if !$run->{upload} && $quiet; - my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}"; + my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}/$run->{media}/"; opendir my $rpmdir, $dir or return; foreach my $rpm (readdir $rpmdir) { my ($rarch, $srpm) = update_srpm($dir, $rpm, $wrong_rpm); @@ -1034,7 +1054,7 @@ sub update_srpm { my ($arch) = $rpm =~ /([^\.]+)\.rpm$/ or return 0; my $srpm = $cache->{rpm_srpm}{$rpm}; if (!$srpm) { - my $hdr = rpm2header("$dir/$rpm"); + my $hdr = RPM4::Header->new("$dir/$rpm"); $hdr or return 0; $srpm = $hdr->queryformat("%{SOURCERPM}"); $cache->{rpm_srpm}{$rpm} = $srpm @@ -1058,7 +1078,7 @@ sub dump_cache { if (-f $filename) { print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1; $old_cache = do $cachefile; - foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning') { + foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning', 'buildrequires') { foreach my $rpm (%{$old_cache->{$k}}) { $cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm} } @@ -1197,7 +1217,7 @@ sub perform_command { print {$run{LOG}} "$command\n" if $run{verbose} > 2; if ($opt{log}) { #$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`; - system("$command 2>&1 >> $logfile"); + system("$command &> $logfile"); } else { $output = `$command 2>&1`; } @@ -1506,7 +1526,7 @@ sub search_packages { my $check_needed = check_needed($srpm, $cache, $provides); $run->{status}{$srpm} = 'missing_buildrequires' if !$check_needed; if (!$cache->{queue}{$srpm} && $check_needed) { - my $hdr = rpm2header("$dir/$srpm"); + my $hdr = RPM4::Header->new("$dir/$srpm"); if (!check_arch($hdr)) { $run->{status}{$srpm} = 'not_on_this_arch'; next @@ -1874,12 +1894,31 @@ sub send_status_mail { } } elsif ($run->{status}{$rpm} eq 'build_failure') { my ($maint) = get_maint($run, $rpm); - push @{$output{build}{$maint}}, $rpm + if ($cache->{buildrequires}{$rpm}) { + push @{$output{buildrequires}{$maint}}, $rpm + } else { + push @{$output{build}{$maint}}, $rpm + } } elsif (!$run->{status}{$rpm}) { # need to find something more usefull to do at that point next } } + my $text = "*** Missing buildrequires tag in specfile ***\n"; + foreach my $maint (keys %{$output{buildrequires}}) { + $text .= "\n$maint\n"; + foreach my $pack (keys %{$output{missing}{$maint}}) { + foreach my $missing (keys %{$cache->{buildrequires}{$pack}}) { + my $rpms = $cache->{buildrequires}{$pack}{$missing}; + if (@$rpms) { + $text .= " $pack should have a buildrequires on @$rpms (for $missing-devel)\n" + } else { + $text .= " $pack should have a buildrequires for $missing-devel\n" + } + } + } + } + my $text = "*** Missing dependencies ***\n"; foreach my $maint (keys %{$output{missing}}) { $text .= "\n$maint\n"; @@ -1920,3 +1959,14 @@ sub get_maint { $run{maint}{$srpm} = $maint; $maint, $srpm_name } + +sub find_provides { + my ($run, $pack_provide, $p) = @_; + my @rpm; + foreach my $provides (keys %{pack_provide}) { + if ($provides =~ /$p/ && $provides =~ /devel/) { + push @rpm, $pack_provide->{$provides}; + } + } + @rpm +} -- cgit v1.2.1