aboutsummaryrefslogtreecommitdiffstats
path: root/iurt2
diff options
context:
space:
mode:
authorFlorent Villard <warly@mandriva.com>2006-06-18 10:05:01 +0000
committerFlorent Villard <warly@mandriva.com>2006-06-18 10:05:01 +0000
commit6b6e6303373488f5b5148c80955555a8c897b946 (patch)
tree51ad33ac910a0c33d1eef7e510662346045389ed /iurt2
parent3acd348abbd511560afb6442a9249e19876ea68f (diff)
downloadiurt-6b6e6303373488f5b5148c80955555a8c897b946.tar
iurt-6b6e6303373488f5b5148c80955555a8c897b946.tar.gz
iurt-6b6e6303373488f5b5148c80955555a8c897b946.tar.bz2
iurt-6b6e6303373488f5b5148c80955555a8c897b946.tar.xz
iurt-6b6e6303373488f5b5148c80955555a8c897b946.zip
- add a new status option to generate summary reports
- create a get_maint function to factorise some code - improve missing dependency checking code. Now detect intermediate package which needs to be recompiled (do not recompile them, though, iurt would need to use rpmbuildupdate for that) - kill zombie iurt process when starting
Diffstat (limited to 'iurt2')
-rwxr-xr-xiurt2184
1 files changed, 141 insertions, 43 deletions
diff --git a/iurt2 b/iurt2
index 36d51e8..5aae5cf 100755
--- a/iurt2
+++ b/iurt2
@@ -51,7 +51,7 @@ $run{todo} = [ ];
[ "", "$program_name", 0, "[--cache] [--chrooted-urpmi <media prefix>] [--concurrent-run] [--config foo value] [--warn] [--verbose integer]
[--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}]
[--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs]
- [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs]
+ [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs] [--status]
{--config_help |
--chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } |
--rebuild {cooker|2006.0|community/2006.0|...} {i586|x86_64|ppc|...} {filename1.src.rpm} {filename2.src.rpm} ... {filenamen.src.rpm} }",
@@ -186,6 +186,12 @@ $run{todo} = [ ];
($run{stop}) = @_;
1
}, "Setting rpm build option" ],
+ [ "", "status", 1, "<mail>",
+ "Send a status mail to the provided mail address",
+ sub {
+ ($run{status_mail}) = @_;
+ 1
+ }, "Setting status mail option" ],
);
open(my $LOG, ">&STDERR");
@@ -339,7 +345,7 @@ $run{cache} = $cache;
my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint);
$to_compile = @{$run{todo}};
$to_compile += check_media(\%run, $cache, $config, \%srpm_version, \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media};
-$to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{extra};
+$to_compile += search_packages(1, $cache, \%provides, \%run, \%maint, \%srpm_version, @{$run{extra_dir}}) if $run{extra};
dump_cache(\%run);
@@ -399,7 +405,7 @@ $chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}/$chroot_name.$run{run
# now exit if there is nothing to do and it was just a cleaning pass
if (!@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild}) {
- print {$run{LOG}} "iurt: nothing to do\n";
+ print {$run{LOG}} "iurt: no package to compile :(\n";
unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
exit
}
@@ -472,14 +478,13 @@ if ($run{group}) {
order_packages(\%run) or die "FATAL iurt: could not order packages";
$local_media = "$local_spool/$run{run}/"
}
-
do {
foreach (my $i ; $i < @{$run{todo}}; $i++) {
my ($dir, $srpm, $status) = @{$run{todo}[$i]};
$status or next;
$done{$srpm} and next;
$done{$srpm} = 1;
- check_version($srpm) or next;
+ check_version($srpm, \%srpm_version) or next;
if ($run{debug}) { $run{debug}++ == 2 and exit }
$done++;
mkdir "$local_spool/log/$srpm";
@@ -529,7 +534,7 @@ do {
my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
my ($maintainer, $cc);
if (!$run{warn}) {
- $maintainer = `rpmmon -s -p $srpm_name`;
+ $maintainer = get_maint(\%run, $srpm);
$cc = "$maint{$srpm}";#, maintainers\@mandriva.com";
chomp $maintainer;
if (!$maintainer || $maintainer eq 'NOT_FOUND') {
@@ -567,7 +572,7 @@ do {
mail => $config->{admin},
error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}",
hash => "install_deps_$srpm",
- timeout => 600,
+ timeout => 900,
srpm => $srpm,
freq => 1,
#cc => $cc,
@@ -579,32 +584,52 @@ do {
'is needed by' => sub {
print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n";
system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null");
- foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } },
+ foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
+ 1
+ },
},
- wait_callback => $wait_urpmi,
log => "$local_spool/log/$srpm/",
callback => sub {
my ($opt, $output) = @_;
print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
# 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error).
# my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g;
- my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
+ my @missing_deps = $output =~ /([^ \n]+) \(due to unsatisfied ([^[ \n]*)(?: ([^\n]*)|\[([^\n]*)\])?\)/g;
# as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
@missing_deps or return 1;
- while (my $missing_deps = shift @missing_deps) {
+ while (my $missing_package = shift @missing_deps) {
+ my $missing_deps = shift @missing_deps;
my $version = shift @missing_deps;
my $version2 = shift @missing_deps;
$version ||= $version2 || 0;
my $p = $pack_provide{$missing_deps} || $missing_deps;
- my $other_maint = `rpmmon -p "$p"`;
- chomp $other_maint;
- print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
- if ($other_maint && $other_maint ne 'NOT_FOUND') {
- $opt->{mail} = "$maintainer, $other_maint";
- $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
+ my ($missing_package_name, $first_maint);
+ if ($missing_package !~ /\.src$/) {
+ ($missing_package_name) = $missing_package =~ /(.*)-[^-]+-[^-]+\.[^.]+$/;
+ ($first_maint, $missing_package_name) = get_maint(\%run, $missing_package);
+ print {$run{LOG}} "iurt: likely $missing_package_name need to be rebuilt ($first_maint)\n" if $run{verbose} > 4;
+ } else {
+ $missing_package = ''
+ }
+ my $other_maint = get_maint(\%run, $p);
+ print {$run{LOG}} "missing dep: $missing_deps ($other_maint) missing_pacakge $missing_package ($first_maint)\n";
+ foreach my $m ($first_maint, $other_maint) {
+ if ($other_maint && $other_maint ne 'NOT_FOUND') {
+ $opt->{mail} = "warly\@mandriva.com";
+ #$opt->{mail} .= ", $other_maint";
+ }
+ }
+ if (!$opt->{mail}) {
+ $opt->{mail} = "warly\@mandriva.com";
}
# remember what is needed, and do not try to recompile until it is available
- $cache->{needed}{$srpm}{$missing_deps} = { version => $version, maint => $other_maint || $maintainer };
+ if ($missing_package) {
+ $opt->{error} = "[MISSING] $missing_deps, needed by $missing_package to build $srpm, is not available on $run{my_arch} (rebuild $missing_package?)";
+ $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $first_maint || $other_maint || $maintainer };
+ } else {
+ $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
+ $cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $other_maint || $maintainer };
+ }
}
0
},
@@ -744,6 +769,8 @@ if (open my $file, ">$local_spool/log/wrong_srpm_names.log") {
dump_status($local_spool, \%run);
+send_status_mail(\%run, $config, $cache) if ($run{status_mail});
+
if ($config->{rsync_to} && !$run{no_rsync}) {
# remove some old and very big log files not to saturate the server
system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|);
@@ -926,12 +953,14 @@ sub check_mounted {
}
sub check_needed {
- my ($srpm) = @_;
+ my ($srpm, $cache, $provides) = @_;
if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 }
my $ok = 1;
foreach my $name (keys %{$cache->{needed}{$srpm}}) {
- my ($version, $maint) = $cache->{needed}{$srpm}{'version','maint'};
- my ($p_version) = $provides{$name};
+ my ($package, $version, $maint) = @{$cache->{needed}{$srpm}{$name}}{'package', 'version','maint'};
+ # if packages does not exist anymore, it may have been rebuild, then try to recompute the build dependencies
+ last if $package && !$provides->{$package};
+ my ($p_version) = $provides->{$name};
if ($p_version) {
next if $version == $p_version;
next if URPM::ranges_overlap($version, $p_version)
@@ -941,12 +970,16 @@ sub check_needed {
$cache->{needed}{$srpm}{$name}{version} = $version;
}
my $v ||= $version;
- print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n";
+ if ($package) {
+ print {$run{LOG}} "ERROR iurt: $srpm needs package $package which requires missing $name $v to be compiled.\n";
+ } else {
+ print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n";
+ }
# try to recompile it once in a while
last if $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72;
return 1
}
- delete $cache->{needed}{$srpm};
+ delete $cache->{needed}{$srpm} if $ok;
$ok
}
@@ -1067,10 +1100,10 @@ sub check_arch {
}
sub check_version {
- my ($srpm) = @_;
+ my ($srpm, $srpm_version) = @_;
my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm/;
- if (URPM::ranges_overlap("= $srpm",">= $srpm_version{$srpm_name}")) {
- $srpm_version{$srpm_name} = $srpm;
+ if (URPM::ranges_overlap("= $srpm",">= $srpm_version->{$srpm_name}")) {
+ $srpm_version->{$srpm_name} = $srpm;
return 1
}
0
@@ -1187,7 +1220,7 @@ sub perform_command {
if ($opt{wait_regexp}) {
foreach my $wr (keys %{$opt{wait_regexp}}) {
if ($output =~ /$wr/m) {
- $inc = $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
+ $inc = $opt{wait_regexp}{$wr}(\%opt, $output) if ref $opt{wait_regexp}{$wr};
print {$run->{LOG}} "ERROR iurt: $wr !\n";
sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$wr\n\n$comment\n$output", 0, 0, $opt{debug_mail});
}
@@ -1202,6 +1235,7 @@ sub perform_command {
}
}
if (!$call_ret || $kill || $err || $opt{error_regexp} && $fulloutput =~ /$opt{error_regexp}/) {
+ print {$run->{LOG}} "ERROR iurt: call_ret $call_ret kill $kill err $err ($opt{error_regexp})\n" if $run->{verbose} > 4;;
if ($opt{log} && $config->{log_url}) {
$comment = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$comment|
}
@@ -1342,8 +1376,10 @@ sub check_pid {
}
if ($pid && getpgrp $pid != -1) {
my $time = $stat[9];
- if ($time < time - 36000) {
- print {$run->{LOG}} "iurt: an other iurt pid $pid is running for a very long time, killing it\n";
+ my $state = `ps h -o state $pid`;
+ chomp $state;
+ if ($time < time - 36000 || $state eq 'Z') {
+ print {$run->{LOG}} "iurt: an other iurt pid $pid is running for a very long time or is zombie, killing it\n";
my $i;
while ($i < 5 && getpgrp $pid != -1) {
kill_for_good($pid);
@@ -1383,7 +1419,7 @@ sub check_media {
$rarch or next;
$cache->{queue}{$srpm} = 1;
$run{status}{$srpm} = 'ok';
- check_version($srpm)
+ check_version($srpm, $srpm_version)
}
closedir $rpmdir;
@@ -1403,7 +1439,9 @@ sub check_media {
}
} elsif (/\@info\@([^@]+)@/) {
$nb++;
- my ($name) = $1 =~ /(.*)-[^-]+-[^-]+\..*$/;
+ my $p = $1;
+ my ($name) = $p =~ /(.*)-[^-]+-[^-]+\..*$/;
+ $provides->{$p} = 1;
foreach (@prov) {
$pack_provide->{$_} = $name
}
@@ -1416,12 +1454,13 @@ sub check_media {
}
}
}
- search_packages(0, $cache, $run, $maint, "$config->{repository}/$run->{distro}/SRPMS/$run->{media}/")
+ search_packages(0, $cache, $provides, $run, $maint, $srpm_version, "$config->{repository}/$run->{distro}/SRPMS/$run->{media}/")
}
sub search_packages {
- my ($clean, $cache, $run, $maint, @dir) = @_;
+ my ($clean, $cache, $provides, $run, $maint, $srpm_version, @dir) = @_;
my ($to_compile, %rep, %done_rpm);
+ print {$run->{LOG}} "iurt search_package: @dir\n";
foreach my $dir (@dir) {
print {$run->{LOG}} "iurt: checking SRPMS dir $dir\n";
opendir my $rpmdir, $dir or next;
@@ -1436,12 +1475,12 @@ sub search_packages {
$run->{status}{$srpm} ||= 0;
if ($config->{unwanted_packages} && $srpm =~ /$config->{unwanted_packages}/) { next }
my $ok = 1;
- if (check_version($srpm)) {
+ if (check_version($srpm, $srpm_version)) {
if (defined $cache->{failure}{$srpm}) {
$run->{status}{$srpm} = 'build_failure';
next
}
- my $check_needed = check_needed($srpm);
+ my $check_needed = check_needed($srpm, $cache, $provides);
$run->{status}{$srpm} = 'missing_buildrequires' if !$check_needed;
if (!$cache->{queue}{$srpm} && $check_needed) {
my $hdr = rpm2header("$dir/$srpm");
@@ -1678,23 +1717,18 @@ sub recreate_srpm {
debug_mail => $run->{debug},
hash => "install_$srpm",
retry => $retry,
- wait_regexp => {
- 'cannot write to %sourcedir /usr/src/rpm/SOURCES' => sub {
- print {$run->{LOG}} "ERROR iurt: chroot seems corrupted, try to rebuild it\n";
- check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run)
- }
- },
callback => sub {
my ($opt, $output) = @_;
print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug};
- if ($output =~ /warning: (group|user) .* does not exist - using root/i) {
+ if ($output =~ /warning: (group|user) .* does not exist - using root|Header V3 DSA signature/i) {
return 1
- } elsif ($output =~ /user $luser does not exist/) {
+ } elsif ($output =~ /user $luser does not exist|cannot write to \%sourcedir/) {
print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1;
$opt->{error} = "[CHROOT] chroot is corrupted";
$opt->{retry} = 1 if !$opt->{retry};
return
}
+ 1
});
print {$run->{LOG}} "iurt: recreating src.rpm...\n" if $run->{verbose};
if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"},
@@ -1799,3 +1833,67 @@ sub dump_status {
}
}
+sub send_status_mail {
+ my ($run, $config, $cache) = @_;
+ print "iurt compilation status\n";
+ my %output;
+ foreach my $rpm (keys %{$run->{status}}) {
+ next if $run->{status}{$rpm} =~ /ok|not_on_this_arch/;
+ if ($run->{status}{$rpm} eq 'missing_buildrequires') {
+ foreach my $missing (keys %{$cache->{needed}{$rpm}}) {
+ my $h = $cache->{needed}{$rpm}{$missing};
+ my $maint = $h->{maint} || 'Other';
+ my $package = $h->{package};
+ if ($package) {
+ push @{$output{missing}{$maint}{$package}{$missing}{$h->{version}}}, $rpm
+ } else {
+ $output{missing}{$maint}{$rpm}{$missing}{$h->{version}} = 1
+ }
+ }
+ } elsif ($run->{status}{$rpm} eq 'build_failure') {
+ my ($maint) = get_maint($run, $rpm);
+ push @{$output{build}{$maint}}, $rpm
+ } elsif (!$run->{status}{$rpm}) {
+ # need to find something more usefull to do at that point
+ next
+ }
+ }
+ my $text = "*** Missing dependencies ***\n";
+ foreach my $maint (keys %{$output{missing}}) {
+ $text .= "\n$maint\n";
+ foreach my $pack (keys %{$output{missing}{$maint}}) {
+ foreach my $missing (%{$output{missing}{$maint}{$pack}}) {
+ my $h = $output{missing}{$maint}{$pack}{$missing};
+ foreach my $version (keys %$h) {
+ if (ref $h->{$version}) {
+ $text .= " $pack should be recompile because\n $missing ". ($version ? "$version " : ''). "is not provided anymore\n";
+ $text .= " to compile ". join("\n ". @{$h->{$version}}). "\n"
+ } else {
+ $text .= " $pack needs $missing ". ($version ? "$version " : ''). "\n";
+ }
+ }
+ }
+ }
+ }
+ $text .= "\n*** Build failure ***\n";
+ foreach my $maint (keys %{$output{build}}) {
+ $text .= "\n$maint\n";
+ foreach my $rpm (@{$output{build}{$maint}}) {
+ $text .= " $rpm (see $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$rpm/)\n"
+ }
+ }
+ sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", 0, 0, 0)
+}
+
+sub get_maint {
+ my ($run, $srpm) = @_;
+ my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.[^.]+$/;
+ $srpm_name ||= $srpm;
+ if ($run{maint}{$srpm}) {
+ return $run{maint}{$srpm}, $srpm_name
+ }
+ my $maint = `rpmmon -s -p $srpm_name`;
+ chomp $maint;
+ $run{maint}{$srpm} = $maint;
+ $maint, $srpm_name
+}