diff options
| -rwxr-xr-x | iurt2 | 184 |
1 files changed, 141 insertions, 43 deletions
@@ -51,7 +51,7 @@ $run{todo} = [ ]; [ "", "$program_name", 0, "[--cache] [--chrooted-urpmi <media prefix>] [--concurrent-run] [--config foo value] [--warn] [--verbose integer] [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}] [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs] - [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs] + [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs] [--status] {--config_help | --chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } | --rebuild {cooker|2006.0|community/2006.0|...} {i586|x86_64|ppc|...} {filename1.src.rpm} {filename2.src.rpm} ... {filenamen.src.rpm} }", @@ -186,6 +186,12 @@ $run{todo} = [ ]; ($run{stop}) = @_; 1 }, "Setting rpm build option" ], + [ "", "status", 1, "<mail>", + "Send a status mail to the provided mail address", + sub { + ($run{status_mail}) = @_; + 1 + }, "Setting status mail option" ], ); open(my $LOG, ">&STDERR"); @@ -339,7 +345,7 @@ $run{cache} = $cache; my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint); $to_compile = @{$run{todo}}; $to_compile += check_media(\%run, $cache, $config, \%srpm_version, \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media}; -$to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{extra}; +$to_compile += search_packages(1, $cache, \%provides, \%run, \%maint, \%srpm_version, @{$run{extra_dir}}) if $run{extra}; dump_cache(\%run); @@ -399,7 +405,7 @@ $chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}/$chroot_name.$run{run # now exit if there is nothing to do and it was just a cleaning pass if (!@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild}) { - print {$run{LOG}} "iurt: nothing to do\n"; + print {$run{LOG}} "iurt: no package to compile :(\n"; unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile}; exit } @@ -472,14 +478,13 @@ if ($run{group}) { order_packages(\%run) or die "FATAL iurt: could not order packages"; $local_media = "$local_spool/$run{run}/" } - do { foreach (my $i ; $i < @{$run{todo}}; $i++) { my ($dir, $srpm, $status) = @{$run{todo}[$i]}; $status or next; $done{$srpm} and next; $done{$srpm} = 1; - check_version($srpm) or next; + check_version($srpm, \%srpm_version) or next; if ($run{debug}) { $run{debug}++ == 2 and exit } $done++; mkdir "$local_spool/log/$srpm"; @@ -529,7 +534,7 @@ do { my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; my ($maintainer, $cc); if (!$run{warn}) { - $maintainer = `rpmmon -s -p $srpm_name`; + $maintainer = get_maint(\%run, $srpm); $cc = "$maint{$srpm}";#, maintainers\@mandriva.com"; chomp $maintainer; if (!$maintainer || $maintainer eq 'NOT_FOUND') { @@ -567,7 +572,7 @@ do { mail => $config->{admin}, error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", hash => "install_deps_$srpm", - timeout => 600, + timeout => 900, srpm => $srpm, freq => 1, #cc => $cc, @@ -579,32 +584,52 @@ do { 'is needed by' => sub { print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null"); - foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } }, + foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } + 1 + }, }, - wait_callback => $wait_urpmi, log => "$local_spool/log/$srpm/", callback => sub { my ($opt, $output) = @_; print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug}; # 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error). # my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g; - my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g; + my @missing_deps = $output =~ /([^ \n]+) \(due to unsatisfied ([^[ \n]*)(?: ([^\n]*)|\[([^\n]*)\])?\)/g; # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway @missing_deps or return 1; - while (my $missing_deps = shift @missing_deps) { + while (my $missing_package = shift @missing_deps) { + my $missing_deps = shift @missing_deps; my $version = shift @missing_deps; my $version2 = shift @missing_deps; $version ||= $version2 || 0; my $p = $pack_provide{$missing_deps} || $missing_deps; - my $other_maint = `rpmmon -p "$p"`; - chomp $other_maint; - print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n"; - if ($other_maint && $other_maint ne 'NOT_FOUND') { - $opt->{mail} = "$maintainer, $other_maint"; - $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}"; + my ($missing_package_name, $first_maint); + if ($missing_package !~ /\.src$/) { + ($missing_package_name) = $missing_package =~ /(.*)-[^-]+-[^-]+\.[^.]+$/; + ($first_maint, $missing_package_name) = get_maint(\%run, $missing_package); + print {$run{LOG}} "iurt: likely $missing_package_name need to be rebuilt ($first_maint)\n" if $run{verbose} > 4; + } else { + $missing_package = '' + } + my $other_maint = get_maint(\%run, $p); + print {$run{LOG}} "missing dep: $missing_deps ($other_maint) missing_pacakge $missing_package ($first_maint)\n"; + foreach my $m ($first_maint, $other_maint) { + if ($other_maint && $other_maint ne 'NOT_FOUND') { + $opt->{mail} = "warly\@mandriva.com"; + #$opt->{mail} .= ", $other_maint"; + } + } + if (!$opt->{mail}) { + $opt->{mail} = "warly\@mandriva.com"; } # remember what is needed, and do not try to recompile until it is available - $cache->{needed}{$srpm}{$missing_deps} = { version => $version, maint => $other_maint || $maintainer }; + if ($missing_package) { + $opt->{error} = "[MISSING] $missing_deps, needed by $missing_package to build $srpm, is not available on $run{my_arch} (rebuild $missing_package?)"; + $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $first_maint || $other_maint || $maintainer }; + } else { + $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}"; + $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $other_maint || $maintainer }; + } } 0 }, @@ -744,6 +769,8 @@ if (open my $file, ">$local_spool/log/wrong_srpm_names.log") { dump_status($local_spool, \%run); +send_status_mail(\%run, $config, $cache) if ($run{status_mail}); + if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|); @@ -926,12 +953,14 @@ sub check_mounted { } sub check_needed { - my ($srpm) = @_; + my ($srpm, $cache, $provides) = @_; if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 } my $ok = 1; foreach my $name (keys %{$cache->{needed}{$srpm}}) { - my ($version, $maint) = $cache->{needed}{$srpm}{'version','maint'}; - my ($p_version) = $provides{$name}; + my ($package, $version, $maint) = @{$cache->{needed}{$srpm}{$name}}{'package', 'version','maint'}; + # if packages does not exist anymore, it may have been rebuild, then try to recompute the build dependencies + last if $package && !$provides->{$package}; + my ($p_version) = $provides->{$name}; if ($p_version) { next if $version == $p_version; next if URPM::ranges_overlap($version, $p_version) @@ -941,12 +970,16 @@ sub check_needed { $cache->{needed}{$srpm}{$name}{version} = $version; } my $v ||= $version; - print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n"; + if ($package) { + print {$run{LOG}} "ERROR iurt: $srpm needs package $package which requires missing $name $v to be compiled.\n"; + } else { + print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n"; + } # try to recompile it once in a while last if $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72; return 1 } - delete $cache->{needed}{$srpm}; + delete $cache->{needed}{$srpm} if $ok; $ok } @@ -1067,10 +1100,10 @@ sub check_arch { } sub check_version { - my ($srpm) = @_; + my ($srpm, $srpm_version) = @_; my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm/; - if (URPM::ranges_overlap("= $srpm",">= $srpm_version{$srpm_name}")) { - $srpm_version{$srpm_name} = $srpm; + if (URPM::ranges_overlap("= $srpm",">= $srpm_version->{$srpm_name}")) { + $srpm_version->{$srpm_name} = $srpm; return 1 } 0 @@ -1187,7 +1220,7 @@ sub perform_command { if ($opt{wait_regexp}) { foreach my $wr (keys %{$opt{wait_regexp}}) { if ($output =~ /$wr/m) { - $inc = $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback}; + $inc = $opt{wait_regexp}{$wr}(\%opt, $output) if ref $opt{wait_regexp}{$wr}; print {$run->{LOG}} "ERROR iurt: $wr !\n"; sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$wr\n\n$comment\n$output", 0, 0, $opt{debug_mail}); } @@ -1202,6 +1235,7 @@ sub perform_command { } } if (!$call_ret || $kill || $err || $opt{error_regexp} && $fulloutput =~ /$opt{error_regexp}/) { + print {$run->{LOG}} "ERROR iurt: call_ret $call_ret kill $kill err $err ($opt{error_regexp})\n" if $run->{verbose} > 4;; if ($opt{log} && $config->{log_url}) { $comment = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$comment| } @@ -1342,8 +1376,10 @@ sub check_pid { } if ($pid && getpgrp $pid != -1) { my $time = $stat[9]; - if ($time < time - 36000) { - print {$run->{LOG}} "iurt: an other iurt pid $pid is running for a very long time, killing it\n"; + my $state = `ps h -o state $pid`; + chomp $state; + if ($time < time - 36000 || $state eq 'Z') { + print {$run->{LOG}} "iurt: an other iurt pid $pid is running for a very long time or is zombie, killing it\n"; my $i; while ($i < 5 && getpgrp $pid != -1) { kill_for_good($pid); @@ -1383,7 +1419,7 @@ sub check_media { $rarch or next; $cache->{queue}{$srpm} = 1; $run{status}{$srpm} = 'ok'; - check_version($srpm) + check_version($srpm, $srpm_version) } closedir $rpmdir; @@ -1403,7 +1439,9 @@ sub check_media { } } elsif (/\@info\@([^@]+)@/) { $nb++; - my ($name) = $1 =~ /(.*)-[^-]+-[^-]+\..*$/; + my $p = $1; + my ($name) = $p =~ /(.*)-[^-]+-[^-]+\..*$/; + $provides->{$p} = 1; foreach (@prov) { $pack_provide->{$_} = $name } @@ -1416,12 +1454,13 @@ sub check_media { } } } - search_packages(0, $cache, $run, $maint, "$config->{repository}/$run->{distro}/SRPMS/$run->{media}/") + search_packages(0, $cache, $provides, $run, $maint, $srpm_version, "$config->{repository}/$run->{distro}/SRPMS/$run->{media}/") } sub search_packages { - my ($clean, $cache, $run, $maint, @dir) = @_; + my ($clean, $cache, $provides, $run, $maint, $srpm_version, @dir) = @_; my ($to_compile, %rep, %done_rpm); + print {$run->{LOG}} "iurt search_package: @dir\n"; foreach my $dir (@dir) { print {$run->{LOG}} "iurt: checking SRPMS dir $dir\n"; opendir my $rpmdir, $dir or next; @@ -1436,12 +1475,12 @@ sub search_packages { $run->{status}{$srpm} ||= 0; if ($config->{unwanted_packages} && $srpm =~ /$config->{unwanted_packages}/) { next } my $ok = 1; - if (check_version($srpm)) { + if (check_version($srpm, $srpm_version)) { if (defined $cache->{failure}{$srpm}) { $run->{status}{$srpm} = 'build_failure'; next } - my $check_needed = check_needed($srpm); + my $check_needed = check_needed($srpm, $cache, $provides); $run->{status}{$srpm} = 'missing_buildrequires' if !$check_needed; if (!$cache->{queue}{$srpm} && $check_needed) { my $hdr = rpm2header("$dir/$srpm"); @@ -1678,23 +1717,18 @@ sub recreate_srpm { debug_mail => $run->{debug}, hash => "install_$srpm", retry => $retry, - wait_regexp => { - 'cannot write to %sourcedir /usr/src/rpm/SOURCES' => sub { - print {$run->{LOG}} "ERROR iurt: chroot seems corrupted, try to rebuild it\n"; - check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) - } - }, callback => sub { my ($opt, $output) = @_; print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug}; - if ($output =~ /warning: (group|user) .* does not exist - using root/i) { + if ($output =~ /warning: (group|user) .* does not exist - using root|Header V3 DSA signature/i) { return 1 - } elsif ($output =~ /user $luser does not exist/) { + } elsif ($output =~ /user $luser does not exist|cannot write to \%sourcedir/) { print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1; $opt->{error} = "[CHROOT] chroot is corrupted"; $opt->{retry} = 1 if !$opt->{retry}; return } + 1 }); print {$run->{LOG}} "iurt: recreating src.rpm...\n" if $run->{verbose}; if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"}, @@ -1799,3 +1833,67 @@ sub dump_status { } } +sub send_status_mail { + my ($run, $config, $cache) = @_; + print "iurt compilation status\n"; + my %output; + foreach my $rpm (keys %{$run->{status}}) { + next if $run->{status}{$rpm} =~ /ok|not_on_this_arch/; + if ($run->{status}{$rpm} eq 'missing_buildrequires') { + foreach my $missing (keys %{$cache->{needed}{$rpm}}) { + my $h = $cache->{needed}{$rpm}{$missing}; + my $maint = $h->{maint} || 'Other'; + my $package = $h->{package}; + if ($package) { + push @{$output{missing}{$maint}{$package}{$missing}{$h->{version}}}, $rpm + } else { + $output{missing}{$maint}{$rpm}{$missing}{$h->{version}} = 1 + } + } + } elsif ($run->{status}{$rpm} eq 'build_failure') { + my ($maint) = get_maint($run, $rpm); + push @{$output{build}{$maint}}, $rpm + } elsif (!$run->{status}{$rpm}) { + # need to find something more usefull to do at that point + next + } + } + my $text = "*** Missing dependencies ***\n"; + foreach my $maint (keys %{$output{missing}}) { + $text .= "\n$maint\n"; + foreach my $pack (keys %{$output{missing}{$maint}}) { + foreach my $missing (%{$output{missing}{$maint}{$pack}}) { + my $h = $output{missing}{$maint}{$pack}{$missing}; + foreach my $version (keys %$h) { + if (ref $h->{$version}) { + $text .= " $pack should be recompile because\n $missing ". ($version ? "$version " : ''). "is not provided anymore\n"; + $text .= " to compile ". join("\n ". @{$h->{$version}}). "\n" + } else { + $text .= " $pack needs $missing ". ($version ? "$version " : ''). "\n"; + } + } + } + } + } + $text .= "\n*** Build failure ***\n"; + foreach my $maint (keys %{$output{build}}) { + $text .= "\n$maint\n"; + foreach my $rpm (@{$output{build}{$maint}}) { + $text .= " $rpm (see $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$rpm/)\n" + } + } + sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", 0, 0, 0) +} + +sub get_maint { + my ($run, $srpm) = @_; + my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.[^.]+$/; + $srpm_name ||= $srpm; + if ($run{maint}{$srpm}) { + return $run{maint}{$srpm}, $srpm_name + } + my $maint = `rpmmon -s -p $srpm_name`; + chomp $maint; + $run{maint}{$srpm} = $maint; + $maint, $srpm_name +} |