diff options
| -rwxr-xr-x | iurt2 | 133 |
1 files changed, 94 insertions, 39 deletions
@@ -24,6 +24,7 @@ # TODO # # - use a cache (rpmctl cache for example) to find maintainer +# - Do not run if the hdlist are broken # use strict; use Hdlist; @@ -48,7 +49,7 @@ $distro_tag =~ s,/,-,g; my $my_arch = shift @argv; my $media = shift @argv; my @special_srpm_dir = @argv; -my $urpmi_options = "-v --no-verify-rpm -s --auto"; +my $urpmi_options = "-v --no-verify-rpm --nolock --auto"; $my_arch or usage("no architecture given"); my $real_arch = `uname -m`; @@ -141,7 +142,7 @@ my $cache; if (-f $cachefile) { $cache = do $cachefile } else { - $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {} } + $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {}, no_unionfs => {} } } my %big; @@ -247,7 +248,6 @@ foreach my $dir ("$config->{repository}/$distro_version/SRPMS/$media/", @special closedir $rpmdir } - dump_cache(); if (!@todo && !$debug) { @@ -260,6 +260,14 @@ print "iurt: will try to compile $to_compile packages\n"; $cache->{run} ||= 1; my $run = ++$cache->{run}; +my $local_spool = "$config->{local_home}/iurt/$distro_tag/$my_arch"; +if (!-d "$config->{local_home}/iurt/$distro_tag/") { + mkdir "$config->{local_home}/iurt/$distro_tag"; + if (!-d $local_spool) { + mkdir $local_spool; + mkdir "$local_spool/log" + } +} my $debug_tag = '_debug' if $debug; my $chroot = "$config->{local_home}/chroot$debug_tag"; @@ -273,6 +281,7 @@ if (!$nocheckchroot) { error => "[REBUILD] Creating the inital chroot for $distro_tag on $my_arch failed", hash => 'chroot_inititialization', timeout => 600, + log => "$local_spool/log/", debug_mail => $debug, die => 1); @@ -286,16 +295,6 @@ if ($unionfs) { $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag"; -d $unionfs_dir or mkdir $unionfs_dir } - -my $local_spool = "$config->{local_home}/iurt/$distro_tag/$my_arch"; -if (!-d "$config->{local_home}/iurt/$distro_tag/") { - mkdir "$config->{local_home}/iurt/$distro_tag"; - if (!-d $local_spool) { - mkdir $local_spool; - mkdir "$local_spool/log" - } -} - my $s = sub { print "iurt: dumping cache...\n"; dump_cache(); exit }; $SIG{TERM} = $s; $SIG{INT} = $s; @@ -313,8 +312,28 @@ foreach (my $i ; $i < @todo; $i++) { if ($debug) { $debug++ == 2 and exit } $done++; print "iurt: packages $srpm [$done/$to_compile]\n"; - if ($unionfs) { + # FIXME unfortunately urpmi stalls quite often + my $match = "urpmi $urpmi_options --root $chroot"; + if (!clean_process($match)) { + dump_cache(); + die "FATAL iurt: Could not have urpmi working !" + } + my $unionfs_tmp; + if ($cache->{no_unionfs}{$srpm}) { + $unionfs_tmp = 0 + } elsif ($unionfs) { + $unionfs_tmp = 1 + } + if ($unionfs_tmp) { + my $mount_point = "$unionfs_dir/unionfs.$run.$union_id"; + print STDERR "Cleaning $mount_point\n"; + if (!clean_mnt($mount_point)) { + dump_cache(); + die "FATAL iurt: could not kill remaining processes acceding $mount_point" + } my $tmpfs; + + # we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database) $union_id = clean_unionfs($unionfs_dir, $run, $union_id); $tmpfs = "$unionfs_dir/tmpfs.$run.$union_id"; $chroot = "$unionfs_dir/unionfs.$run.$union_id"; @@ -328,19 +347,15 @@ foreach (my $i ; $i < @todo; $i++) { } my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; my $maintainer = `rpmmon -s -p $srpm_name`; - my $cc = "$maint{$srpm}, maintainers\@mandriva.com"; + my $cc = "$maint{$srpm}";#, maintainers\@mandriva.com"; chomp $maintainer; if (!$maintainer || $maintainer eq 'NOT_FOUND') { $maintainer = $cc; - $cc = 'maintainers@mandriva.com' + #$cc = 'maintainers@mandriva.com' } #($maintainer, $cc) = ($config->{admin},''); print "Installing build dependencies of $srpm...\n"; - # FIXME unfortunately urpmi stalls quite often - if (!clean_urpmi($home)) { - dump_cache(); - die "FATAL iurt: Could not have urpmi working !" - } + #system(qq{sudo pkill -9 dd-u root -f "$todo[$i-1][1]"}) if $i - 1 >= 0; #system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot"}); perform_command("sudo urpmi $urpmi_options --root $chroot $dir/$srpm", @@ -357,17 +372,17 @@ foreach (my $i ; $i < @todo; $i++) { log => "$local_spool/log/", callback => sub { my ($opt, $output) = @_; - print "Calling callback for $opt->{hash}\n" if $debug; - my ($missing_deps, $version) = $output =~ /\(due to unsatisfied ([^[)]*)(\[.*\])?/; + print "calling callback for $opt->{hash}\n" if $debug; + my ($missing_deps, $version) = $output =~ /\(due to unsatisfied ([^[)]*)(?:\[(.*)\])?/; $missing_deps or return; my $p = $pack_provide{$missing_deps} || $missing_deps; my $other_maint = `rpmmon -p $p`; - $version ||= 1; + $version ||= 0; chomp $other_maint; - print "Missing Dep: $missing_deps ($other_maint)\n"; - if ($other_maint && $other_maint ne 'NOT_FOUND') { + print "missing dep: $missing_deps ($other_maint)\n"; + if ($other_maint && $other_maint ne 'not_found') { $opt->{mail} = $other_maint; - $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $my_arch"; + $opt->{error} = "[missing] $missing_deps, needed to build $srpm, is not available on $my_arch"; } # remember what is needed, and do not try to recompile until it is available push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ]; @@ -395,8 +410,32 @@ foreach (my $i ; $i < @todo; $i++) { cc => $cc, log => "$local_spool/log/", error_regexp => 'rror.*ailed|Bad exit status|RPM build error', + callback => sub { + my ($opt, $output) = @_; + print STDERR "iurt: calling callback for $opt->{hash}\n" if $debug; + if ($unionfs_tmp && $output =~ /no space left on device/i) { + print STDERR "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n"; + $cache->{no_unionfs}{$srpm} = 1 + } + }, freq => 1) && !glob "$chroot/home/builder/rpm/RPMS/*/*.rpm") { - $cache->{failure}{$srpm} = 1; + # FIXME + # The simple algo used here is : + # try to compile it with unionfs, if it runs out of space, compile it without the next time + # + # This could be improved in keeping this srpm name for future version, but if we compile it + # on a new machine with more ram, or if next version compiles just fine with unionfs, we will + # loose the unionfs advantage. + # + # Maybe the right thing to do would be to first try to increate the tmpfs size (more than 50 % of the + # physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs + # plus). Or to keep the faulty package an unionfs exception for some time, to save some more extra builds. + # + if (!$unionfs_tmp) { + $cache->{failure}{$srpm} = 1 + } elsif ($cache->{no_unionfs}{$srpm}) { + $cache->{failure}{$srpm} = 1 + } next } # do some cleaning if the compilation is successful @@ -496,18 +535,26 @@ sub clean_chroot { 1 } -sub clean_urpmi { - my ($chroot) = @_; - my $match = "urpmi $urpmi_options --root $chroot"; +sub clean_process { + my ($match) = @_; + return clean($match, "pgrep -u root -f", "sudo pkill -9 -u root -f"); +} + +sub clean_mnt { + my ($mount_point) = @_; + return clean($mount_point, "fuser", "sudo fuser -k"); +} + +sub clean { + my ($var, $cmd, $kill_cmd) = @_; my $ps; my $i; - $ps = `pgrep -u root -f "$match"`; - while ($ps) { - system(qq{sudo pkill -9 -u root -f "$match"}); + while ($ps = `$cmd "$var"`) { + system(qq{$kill_cmd "$var"}); sleep 1; - $ps = `pgrep -u root -f "$match"`; - print STDERR "Trying to removed previous blocked urpmi session matching $match ($ps)\n"; - $i++ > 10 and return 0 + chomp $ps; + print STDERR "Trying to removed previous blocked processes for $var ($ps)\n"; + return 0 if $i++ > 10 } 1 } @@ -557,6 +604,8 @@ sub check_needed { } $ok = 0; push @n, [ $name, $version ]; + my $v ||= $version; + print STDERR "ERROR iurt: $srpm needs $name $v to be compiled.\n"; # try to recompile it once in a while return 1 if ! $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72 } @@ -675,7 +724,8 @@ sub perform_command { alarm $opt{timeout}; print "$command\n"; if ($opt{log}) { - $output = `$command 2>&1 | tee $opt{log}/$opt{hash}.$run.log`; + #$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`; + system("$command &> $opt{log}/$opt{hash}.$run.log"); } else { $output = `$command 2>&1`; } @@ -686,6 +736,11 @@ sub perform_command { die unless $@ eq "alarm\n"; # propagate unexpected errors return 0 } else { + # Keep the run first on the harddrive so that one can check the command status tailing it + if ($opt{log} && open my $log, "$opt{log}/$opt{hash}.$run.log") { + local $/; + $output = <$log> + } my $err = $?; if ($kill) { $output = "Command has been killed after $opt{timeout} seconds: $command\n$output" @@ -707,7 +762,7 @@ sub perform_command { } if ($err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) { if ($opt{mail} && $config->{sendmail}) { - if (! $cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq}) { + if (! ($cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq})) { sendmail($opt{mail}, $opt{cc} , $opt{error} , $output, 0, 0, $opt{debug_mail}); } elsif ($config->{admin}) { sendmail($config->{admin}, '' , $opt{error}, $output, 0, 0, $opt{debug_mail}); |