From 7a6194c24d910adddd513733d8cd0a14d82e039a Mon Sep 17 00:00:00 2001 From: Florent Villard Date: Fri, 6 Jan 2006 15:31:33 +0000 Subject: - add --unionfs option to use tmpfs+unionfs to build the packages - add --nocheckchroot to directly launch the build without initial basystem tar check - use a different cache per architecture not to have to handle cache merging --- iurt2 | 250 ++++++++++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 182 insertions(+), 68 deletions(-) (limited to 'iurt2') diff --git a/iurt2 b/iurt2 index 973a342..ad78503 100755 --- a/iurt2 +++ b/iurt2 @@ -21,6 +21,12 @@ # # compare and rebuild packages on different architecture # +# TODO +# +# - use a directory + tmpfs via unionfs to build the packages +# - having only one cache for all the architecture would need a merging before the cache is dumped, this is a bit overkill, so several cache are used, as a consequence the rpm/srpm structure is duplicated. Maybe a separated cache could be done to handle this structure, or just have it created separately. Anyway this is just a mater of some hundreds of kilobytes and directory listings. +# - use a cache (rpmctl cache for example) to find maintainer +# use strict; use Hdlist; use Data::Dumper; @@ -29,9 +35,11 @@ use File::NCopy qw(copy); use MIME::Words qw(encode_mimewords); use Fcntl ':flock'; -my ($debug); +my ($debug, $unionfs, $nocheckchroot); my @argv = grep { if (/--debug/) { $debug = 1; 0 } + elsif (/--unionfs/) { $unionfs = 1; 0 } + elsif (/--nocheckchroot/) { $nocheckchroot = 1; 0 } else { 1 } } @ARGV; @@ -41,6 +49,7 @@ $distro_tag =~ s,/,-,g; my $my_arch = shift @argv; my $media = shift @argv; my @special_srpm_dir = @argv; +my $urpmi_options = "-v --no-verify-rpm -s --auto "; $my_arch or usage(); my $real_arch = `uname -m`; @@ -58,7 +67,7 @@ if (!$arch_comp{$real_arch}{$my_arch}) { my $HOME = $ENV{HOME}; my $configfile = "$HOME/.iurt.$distro_tag.conf"; -print "iurt: loading config file $configfile\n"; +print STDERR "iurt: loading config file $configfile\n"; my $config; if (-f $configfile) { $config = do $configfile or die "FATAL iurt: syntax error in $configfile"; @@ -85,7 +94,7 @@ if ($distro_version ne 'cooker') { $config->{upload} .= "/$media" } } elsif ($media eq 'contrib') { - $config->{upload} = "$config->{upload}/contrib" + $config->{upload} =~ s/cooker/contrib/g; } -d $config->{upload} or usage("$config->{upload} does not exist"); @@ -102,7 +111,7 @@ if (!$debug) { if ($pid && getpgrp $pid != -1) { my $time = $stat[9]; if ($time < time - 36000) { - print "iurt: an other iurt pid $pid is running for a very long time, killing it\n"; + print STDERR "iurt: an other iurt pid $pid is running for a very long time, killing it\n"; my $i; while ($i < 5 && getpgrp $pid != -1) { kill_for_good($pid); @@ -110,11 +119,11 @@ if (!$debug) { sleep 1 } } else { - print "iurt: an other iurt is running for $my_arch, pid $pid, since ",time - $time," seconds\n"; + print STDERR "iurt: an other iurt is running for $my_arch, pid $pid, since ",time - $time," seconds\n"; # exit } } else { - print "iurt: a previous iurt for $my_arch seems dead, cleaning.\n"; + print STDERR "iurt: a previous iurt for $my_arch seems dead, cleaning.\n"; unlink $pidfile } } @@ -126,13 +135,13 @@ if (!$debug) { close $lock; } -my $cachefile = "$config->{cache_home}/iurt.$distro_tag.cache"; -print "iurt: loading cache file $cachefile\n"; +my $cachefile = "$config->{cache_home}/iurt.$distro_tag.$my_arch.cache"; +print STDERR "iurt: loading cache file $cachefile\n"; my $cache; if (-f $cachefile) { $cache = do $cachefile } else { - $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => {} } + $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {} } } my %big; @@ -142,18 +151,35 @@ my @wrong_rpm; # in that case, @supported_arch may be removed # #foreach my $arch (@{$config->{supported_arch}}) { - my $rpms_dir = "$config->{repository}/$distro_version/$my_arch/media/$media/"; - print "iurt: checking current packages in $rpms_dir\n"; - opendir my $rpmdir, $rpms_dir or die "Could not open $rpms_dir: $!"; - foreach my $rpm (readdir $rpmdir) { - my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm); - $rarch or next; - $big{$rpm} = 1; - $cache->{queue}{$srpm}{$my_arch} = 1; - $cache->{queue}{$srpm}{$rarch} = 1; - check_version($srpm) - } - closedir $rpmdir; +my $rpms_dir = "$config->{repository}/$distro_version/$my_arch/media/$media/"; +print STDERR "iurt: checking current packages in $rpms_dir\n"; +opendir my $rpmdir, $rpms_dir or die "Could not open $rpms_dir: $!"; +foreach my $rpm (readdir $rpmdir) { + my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm); + $rarch or next; + $big{$rpm} = 1; + $cache->{queue}{$srpm} = 1; + check_version($srpm) +} +closedir $rpmdir; + +my %provides; +my $synthesis_file = "$config->{repository}/$distro_version/$my_arch/media/$media/media_info/synthesis.hdlist.cz"; +if (-f $synthesis_file) { + print STDERR "Parsing $synthesis_file\n"; + if (open my $syn, "zcat $synthesis_file |") { + while (<$syn>) { + if (/^\@provides@(.*)/) { + foreach my $p (split '@', $1) { + $p =~ /([^[]+)(?:\[(.*)\])?/g; + $provides{$1} = $2 || 1 + } + } + } + } else { + print STDERR "ERROR: Could not open $synthesis_file, will not precheck dependecies avaibility\n"; + } +} #} my %maint; @@ -167,6 +193,7 @@ my %done_rpm; # FIXME all the rep but the first one are cleaned # this may not be usefull anymore # +my $to_compile; foreach my $dir ("$config->{repository}/$distro_version/SRPMS/$media/", @special_srpm_dir) { print "iurt: checking SRPMS dir $dir\n"; opendir my $rpmdir, $dir or next; @@ -181,18 +208,19 @@ foreach my $dir ("$config->{repository}/$distro_version/SRPMS/$media/", @special if ($config->{unwanted_packages} && $srpm =~ /$config->{unwanted_packages}/) { next } my $ok = 1; if (check_version($srpm)) { - defined $cache->{failure}{$srpm} && defined $cache->{failure}{$srpm}{$my_arch} and next; - if (!$cache->{queue}{$srpm}{$my_arch} && !$cache->{queue}{$srpm}{noarch}) { + defined $cache->{failure}{$srpm} and next; + if (!$cache->{queue}{$srpm} && check_needed($srpm)) { my $hdr = rpm2header("$dir/$srpm"); check_arch($hdr) and next; my $changelog = $hdr->queryformat("%{CHANGELOGNAME}"); my ($mail) = $changelog =~ /<(.*@.*)>/; $maint{$srpm} = $mail; print "iurt: will try to compile $srpm\n"; + $to_compile++; push @todo, [ $dir , $srpm ] } foreach my $arch (@{$config->{supported_arch}}) { - $ok &&= $cache->{queue}{$srpm}{$arch} || $cache->{queue}{$srpm}{noarch} + $ok &&= $cache->{queue}{$srpm} } } if ($clean && ($rep{$srpm} || $ok)) { @@ -214,54 +242,70 @@ if (!@todo && !$debug) { unlink $pidfile; exit } +print "iurt: will try to compile $to_compile packages\n"; -my $run = $cache->{run}{$distro_tag}{$my_arch}++; +my $run = $cache->{run}++; -print "iurt: checking basesystem tar\n"; my $debug_tag = '_debug' if $debug; my $chroot = "$config->{local_home}/chroot$debug_tag"; my $chroot_tar = "$chroot-$distro_tag.$my_arch.tar.gz"; -perform_command("sudo $config->{install_chroot_binary} cooker $config->{basesystem_media} $chroot_tar $chroot 501 basesystem rpm-build", - mail => $config->{admin}, +if (!$nocheckchroot) { + print "iurt: checking basesystem tar\n"; + system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot"}); + perform_command("sudo $config->{install_chroot_binary} cooker $config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build", + mail => $config->{admin}, error => "[REBUILD] Creating the inital chroot for $distro_tag on $my_arch failed", hash => 'chroot_inititialization', + timeout => 600, debug_mail => $debug, die => 1); + if ($unionfs) { + clean_chroot($chroot) or die "FATAL iurt: Could no prepare initial chroot" + } +} + my $local_spool = "$config->{local_home}/iurt/$distro_tag/$my_arch"; if (!-d "$config->{local_home}/iurt/$distro_tag/") { - mkdir "$config->{local_home}/iurt/$distro_tag"; - if (!-d $local_spool) { - mkdir $local_spool; - mkdir "$local_spool/log" - } + mkdir "$config->{local_home}/iurt/$distro_tag"; + if (!-d $local_spool) { + mkdir $local_spool; + mkdir "$local_spool/log" + } } my %done; -foreach my $t (@todo) { - my ($dir, $srpm) = @$t; +my $wait_limit; +my $done; +my $home = $config->{local_home}; +foreach (my $i ; $i < @todo; $i++) { + my ($dir, $srpm) = @{$todo[$i]}; $done{$srpm} and next; $done{$srpm} = 1; check_version($srpm) or next; if ($debug) { $debug++ == 2 and exit } - print "iurt: installing a new chroot for $srpm in $chroot\n"; - -d $chroot and perform_command("sudo rm -rf $chroot", - mail => $config->{admin}, - error => "[REBUILD] Deleting of old chroot $chroot failed", - hash => 'chroot_deletion', - debug_mail => $debug, - die => 1); - mkdir $chroot; - perform_command("pushd $chroot && sudo tar xvf $chroot_tar", - mail => $config->{admin}, - error => "[REBUILD] creating the initial chroot $chroot failed", - hash => 'chroot_init', - debug_mail => $debug, - die => 1); - - dump_rpmmacros("$chroot/home/builder/.rpmmacros") or next; - + $done++; + # dump cache every ten packages (we can be killed by other iurt if we are stuck) + if (!$done % 10) { dump_cache() } + print "iurt: packages $srpm [$done/$to_compile]\n"; + if ($unionfs) { + foreach my $t ("unionfs",'tmpfs') { + my $d = "$home/$t"; + while (check_mounted($d, $t)) { + print STDERR "iurt: umounting $d\n"; + system(qq{sudo umount $d}) and die "FATAL iurt: could not umount $d" + } + print STDERR "iurt: removing $d\n"; + system(qq{sudo rm -rf $d}); + mkdir "$d"; + } + system(qq{sudo mount -t tmpfs none $home/tmpfs}) and die "FATAL iurt: could not mount $home/tmpfs ($!)"; + system(qq{sudo mount -o dirs=$home/tmpfs=rw:$home/chroot/=ro -t unionfs unionfs $home/unionfs}) and die "FATAL iurt: could not mount $home/tmpfs and $home/chroot with unionfs ($!)"; + } else { + print "iurt: installing a new chroot for $srpm in $chroot\n"; + clean_chroot($chroot) + } my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; my $maintainer = `rpmmon -s -p $srpm_name`; my $cc = "$maint{$srpm}, maintainers\@mandriva.com"; @@ -273,30 +317,35 @@ foreach my $t (@todo) { #($maintainer, $cc) = ($config->{admin},''); print "Installing build dependencies of $srpm...\n"; # FIXME unfortunately urpmi stalls quite often - system(qq{sudo pkill -9 -u root -f "urpmi --root $chroot"}); - perform_command("sudo urpmi -v --root $chroot --no-verify-rpm -s --auto $dir/$srpm", + system(qq{sudo pkill -9 -u root -f "$todo[$i-1][1]"}) if $i - 1 >= 0; + system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --rrot $chroot"}); + perform_command("sudo urpmi $urpmi_options --root $chroot $dir/$srpm", mail => $maintainer, error => "[REBUILD] install of build dependencies of $srpm failed on $my_arch", hash => "install_deps_$srpm", timeout => 600, - freq => 48, + freq => 1, cc => $cc, debug_mail => $debug, error_regexp => 'cannot be installed', wait_regexp => 'database locked', + wait_callback => sub { $wait_limit++; if ($wait_limit > 10) { $wait_limit = 0; system(qq{sudo pkill -9 urpmi}) } }, log => "$local_spool/log/", callback => sub { my ($opt, $output) = @_; print "Calling callback for $opt->{hash}\n" if $debug; - my ($missing_deps) = $output =~ /\(due to unsatisfied ([^[)]*)/; + my ($missing_deps, $version) = $output =~ /\(due to unsatisfied ([^[)]*)(\[.*\])?/; $missing_deps or return; my $other_maint = `rpmmon -p $missing_deps`; + $version ||= 1; + # remember what is needed, and do not try to recompile until it is available chomp $other_maint; print "Missing Dep: $missing_deps ($other_maint)\n"; if ($other_maint) { $opt->{mail} = $other_maint; $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $my_arch"; } + push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ]; }, ) or next; perform_command("sudo chroot $chroot rpm -qa", @@ -322,10 +371,12 @@ foreach my $t (@todo) { log => "$local_spool/log/", error_regexp => 'rror.*ailed|Bad exit status|RPM build error', freq => 1) && !glob "$chroot/home/builder/rpm/RPMS/*/*.rpm") { - $cache->{failure}{$srpm}{$my_arch} = 1; + $cache->{failure}{$srpm} = 1; next } - if (!perform_command("sudo urpmi --root $chroot --no-verify-rpm --auto $chroot/home/builder/rpm/RPMS/*/*.rpm", + # do some cleaning if the compilation is successful + delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm}; + if (!perform_command("sudo urpmi $urpmi_options --root $chroot $chroot/home/builder/rpm/RPMS/*/*.rpm", mail => $maintainer, error => "[REBUILD] binaries packages generated from $srpm do not install correctly", hash => "binary_test_$srpm", @@ -334,7 +385,7 @@ foreach my $t (@todo) { freq => 1, error_regexp => 'unable to access', log => "$local_spool/log/")) { - $cache->{failure}{$srpm}{$my_arch} = 1; + $cache->{failure}{$srpm} = 1; next } if ($debug) { @@ -344,6 +395,7 @@ foreach my $t (@todo) { system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_spool") and print "ERROR: could not copy rpm files from $chroot/home/builder/rpm/RPMS/ to $local_spool ($!)\n"; process_queue() } + end: } print "iurt: reprocess generated packages queue\n"; @@ -385,20 +437,75 @@ sub usage { exit } +sub clean_chroot { + my ($chroot) = @_; + -d $chroot and perform_command("sudo rm -rf $chroot", + mail => $config->{admin}, + error => "[REBUILD] Deleting of old chroot $chroot failed", + hash => 'chroot_deletion', + debug_mail => $debug, + die => 1); + mkdir $chroot; + perform_command("pushd $chroot && sudo tar xvf $chroot_tar", + mail => $config->{admin}, + error => "[REBUILD] creating the initial chroot $chroot failed", + hash => 'chroot_init', + debug_mail => $debug, + die => 1); + + dump_rpmmacros("$chroot/home/builder/.rpmmacros") or return; + 1 +} + +sub check_mounted { + my ($mount_point, $type) = @_; + open my $mount, '/proc/mounts' or die 'FATAL iurt: could not open /proc/mounts'; + $mount_point =~ s,//+,/,g; + while (<$mount>) { + return 1 if /^\w+ $mount_point $type / + } + 0 +} + +sub check_needed { + my ($srpm) = @_; + if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 } + my @n; + my $ok = 1; + foreach my $t (@{$cache->{needed}{$srpm}}) { + my ($name, $version, $maint) = @$t; + my ($p_version) = $provides{$name}; + if ($p_version) { + next if $version == $p_version; + next if URPM::ranges_overlap($version, $p_version) + } + $ok = 0; + push @n, [ $name, $version ]; + # try to recompile it once in a while + return 1 if ! $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72 + } + delete $cache->{needed}{$srpm}; + $cache->{needed}{$srpm} = \@n if @n; + $ok +} + sub process_queue { my $dir = "$config->{local_home}/iurt/$distro_tag/$my_arch"; opendir my $rpmdir, $dir or next; foreach my $rpm (readdir $rpmdir) { my ($rarch, $srpm) = update_srpm($dir, $rpm); $rarch or next; - # try to keep the opportunity to prevent disk full - my $ok = copy "$dir/$rpm", "$config->{upload}/RPMS/"; - if (!$ok){ + # recheck if the package has not been uploaded in the meantime + if (! -f "$rpms_dir/$rpm") { + my $ok = copy "$dir/$rpm", "$config->{upload}/RPMS/"; + # try to keep the opportunity to prevent disk full + if (!$ok){ print "ERROR process_queue: cannot copy $dir/$rpm to $config->{upload}/RPMS/ ($!)\n"; next + } } unlink "$dir/$rpm"; - $cache->{queue}{$srpm}{$rarch} = 1 + $cache->{queue}{$srpm} = 1 } closedir $rpmdir; } @@ -508,22 +615,29 @@ sub perform_command { } else { $output = "Command failed: $command\n$output" } - if ($opt{callback}) { + if (ref $opt{callback}) { $opt{callback}(\%opt, $output) } - if ($output =~ /$opt{wait_regexp}/) { + if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/) { + $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback}; + print STDERR "ERROR iurt: $opt{wait_regexp} !\n"; + sendmail($config->{admin}, '' , "$opt{hash} on $my_arch for $media: could not proceed", "$opt{wait_regexp}\n\n$output", 0, 0, $opt{debug_mail}); + if ($opt{die}) { + dump_cache(); + die "FATAL iurt: $opt{error}." + } return 0 } if ($? || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) { if ($opt{mail} && $config->{sendmail}) { - if (! $cache->{warning}{$opt{hash}}{$my_arch}{$opt{mail}} % $opt{freq}) { + if (! $cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq}) { sendmail($opt{mail}, $opt{cc} , $opt{error} , $output, 0, 0, $opt{debug_mail}); } elsif ($config->{admin}) { sendmail($config->{admin}, '' , $opt{error}, $output, 0, 0, $opt{debug_mail}); } } - $cache->{warning}{$opt{hash}}{$my_arch}{$opt{mail}}++; - print "\n$output\n"; + $cache->{warning}{$opt{hash}}{$opt{mail}}++; + print STDERR "\n$output\n"; if ($opt{die}) { dump_cache(); die "FATAL iurt: $opt{error}." -- cgit v1.2.1