From 60979c8038010f5b5b7f32e9c3b387230a7ffeb7 Mon Sep 17 00:00:00 2001 From: Pascal Terjan Date: Mon, 10 Dec 2012 09:38:12 +0000 Subject: Add support for parallel build --- iurt | 107 +++++++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 32 deletions(-) diff --git a/iurt b/iurt index f258093..885162a 100755 --- a/iurt +++ b/iurt @@ -173,6 +173,9 @@ $run{todo} = []; [ "", "concurrent-run", 0, "", "Allow several iurt to run on different machines (slower)", sub { $run{concurrent_run} = 1 }, "Activating concurrent run checks" ], + [ "", "parallel", 1, "", + "Build up to packages in parallel", + sub {($run{parallel}) = @_; 1}, "Enabling parallel build" ], [ "d", "dir", -1, "", "Directory where to find packages to rebuild", sub { $run{extra_dir} = \@_; 1 }, "Adding extra source packages directories" ], @@ -813,33 +816,16 @@ my $_s = sub { #$SIG{INT} = $s; $run{main} = 1; -my $rebuild; -$run{group} = 0 if @{$run{todo}} == 1; -if ($run{group}) { - $rebuild = 1; - $urpmi->set_local_media($local_spool); - $urpmi->order_packages(\%provides, $luser) - or die "FATAL $program_name: could not order packages"; -} -# -# The build loop -# -my $prev_done = $done; -do { - $rebuild = 0; - $done = $prev_done; - my $i; - for ($i; $i < @{$run{todo}}; $i++) { - my ($dir, $srpm, $status) = @{$run{todo}[$i]}; - +sub rebuild_one { + my ($dir, $srpm, $status) = @_; # CM: Set argv[0] (in the C sense) to something we can easily spot and # understand in process list $0 = "Iurt: $run{distro_tag} $run{my_arch} $run{media} $srpm"; - $status or next; - $done{$srpm} and next; + $status or return; + $done{$srpm} and return; $done{$srpm} = 1; - check_version(\%run, $srpm, \%srpm_version) or next; + check_version(\%run, $srpm, \%srpm_version) or return; if ($run{debug}) { $run{debug}++ == 2 and exit() } $done++; plog('NOTIFY', "Build package $srpm [$done/$to_compile]"); @@ -849,7 +835,7 @@ do { $run{status}{$srpm} = 'missing'; dump_cache_par(\%run); dump_status($local_spool, \%run); - next; + return; } # FIXME unfortunately urpmi stalls quite often my $retry = 0; @@ -864,14 +850,14 @@ retry: if (!$run{use_old_chroot}) { $chroot_tmp = create_temp_chroot(\%run, $config, - $chroot_tmp, $chroot_ref) or next; + $chroot_tmp, $chroot_ref) or return; } if (!$urpmi->urpmi_command($chroot_tmp)) { plog('DEBUG', "Creating chroot failed.\nCommand was: $chroot_tmp"); - next; + return; } - $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; + $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or return; my ($maintainer, $cc); if (!$run{warn}) { ($maintainer) = get_maint(\%run, $srpm); @@ -885,7 +871,7 @@ retry: #($maintainer, $cc) = ($config->{admin},''); plog('DEBUG', "creating user $luser in chroot"); - add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or next; + add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or return; my $old_srpm = $srpm; my ($ret, $spec); @@ -908,7 +894,7 @@ retry: $run{status}{$srpm} = 'recreate_srpm_failure'; dump_cache_par(\%run); dump_status($local_spool, \%run); - next; + return; } (my $log_dirname = $srpm) =~ s/.*:(.*)\.src.rpm/$1/; @@ -928,7 +914,7 @@ retry: my $ok = $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'install_deps', "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", { maintainer => $maintainer }, "$path_srpm/$srpm"); if (!$ok) { $run{status}{$srpm} ||= 'install_deps_failure'; - next; + return; } # try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: @@ -1009,7 +995,7 @@ retry: # 20060615 dump_cache_par(\%run); dump_status($local_spool, \%run); - next; + return; } # do some cleaning if the compilation is successful @@ -1019,7 +1005,7 @@ retry: opendir my $binfh, "$chroot_tmp/home/$luser/rpmbuild/RPMS/"; my @packages; foreach my $bindir (readdir $binfh) { - -d "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir" or next; + -d "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir" or return; opendir my $rpmfh, "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir"; push @packages, map { "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir/$_" } grep { !/src\.rpm$/ && /\.rpm$/ } readdir $rpmfh; } @@ -1028,7 +1014,7 @@ retry: # segfaulting when trying to install packages if ($config->{check_binary_file}) { - $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", { maintainer => $maintainer } ,@packages) or next; + $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", { maintainer => $maintainer } ,@packages) or return; } else { my $successfile = "$local_spool/log/$srpm/binary_test_$srpm-1.log"; open my $f, ">$successfile"; @@ -1066,6 +1052,63 @@ retry: dump_status($local_spool, \%run); # dump_cache each time so that concurrent process can get updated dump_cache_par(\%run) if $run{concurrent_run}; + return 1; +} + +my $rebuild; +$run{group} = 0 if @{$run{todo}} == 1; +if ($run{group}) { + $rebuild = 1; + $urpmi->set_local_media($local_spool); + $urpmi->order_packages(\%provides, $luser) + or die "FATAL $program_name: could not order packages"; +} +# +# The build loop +# +my $prev_done = $done; +do { + $rebuild = 0; + $done = $prev_done; + my $i; + my %children; + for ($i; $i < @{$run{todo}}; $i++) { + my ($dir, $srpm, $status) = @{$run{todo}[$i]}; + if ($run{parallel}) { + my $pid; + # First cleanup all the finished ones + do { + $pid = waitpid(-1, WNOHANG); + if ($pid > 0) { + plog('INFO', "Child $pid has exited"); + delete $children{$pid}; + } + } while $pid > 0; + while (scalar keys %children >= $run{parallel}) { + plog('INFO', "Too many children, waiting to fork more"); + $pid = waitpid(-1, 0); + delete $children{$pid} if $pid > 0; + } + my $pid = fork; + if ($pid) { #parent + $children{$pid} = 1; + } elsif ($pid == 0) { #child + $chroot_tmp .= "_" . int($i); + rebuild_one($dir, $srpm, $status); + exit; + } else { + die "could not fork"; + } + } else { + rebuild_one($dir, $srpm, $status); + } + } + if ($run{parallel}) { + foreach my $pid (keys %children) { + plog('INFO', "Waiting for process $pid to exit"); + waitpid($pid, 0); + delete $children{$pid}; + } } if ($run{group}) { my $i; -- cgit v1.2.1