Add support for parallel build

1 files changed, 75 insertions, 32 deletions

diff --git a/iurt b/iurt
index f258093..885162a 100755
--- a/iurt
+++ b/iurt
@@ -173,6 +173,9 @@ $run{todo} = [];
     [ "", "concurrent-run", 0, "", 
     "Allow several iurt to run on different machines (slower)",
     sub { $run{concurrent_run} = 1 }, "Activating concurrent run checks" ],
+    [ "", "parallel", 1, "<n>",
+    "Build up to <n> packages in parallel",
+    sub {($run{parallel}) = @_; 1}, "Enabling parallel build" ],
     [ "d", "dir", -1, "", 
     "Directory where to find packages to rebuild", 
     sub { $run{extra_dir} = \@_; 1 }, "Adding extra source packages directories" ],
@@ -813,33 +816,16 @@ my $_s = sub {
 #$SIG{INT} = $s;
 $run{main} = 1;
 
-my $rebuild;
-$run{group} = 0 if @{$run{todo}} == 1;
-if ($run{group}) { 
-    $rebuild = 1;
-    $urpmi->set_local_media($local_spool);
-    $urpmi->order_packages(\%provides, $luser)
-	or die "FATAL $program_name: could not order packages";
-}
-#
-# The build loop
-#
-my $prev_done = $done;
-do { 
-    $rebuild = 0;
-    $done = $prev_done;
-    my $i;
-    for ($i; $i < @{$run{todo}}; $i++) {
-	my ($dir, $srpm, $status) = @{$run{todo}[$i]};
-	
+sub rebuild_one {
+	my ($dir, $srpm, $status) = @_;
 	# CM: Set argv[0] (in the C sense) to something we can easily spot and
 	#     understand in process list
 	$0 = "Iurt: $run{distro_tag} $run{my_arch} $run{media} $srpm";
 
-	$status or next;
-	$done{$srpm} and next;
+	$status or return;
+	$done{$srpm} and return;
 	$done{$srpm} = 1;
-	check_version(\%run, $srpm, \%srpm_version) or next;
+	check_version(\%run, $srpm, \%srpm_version) or return;
 	if ($run{debug}) { $run{debug}++ == 2 and exit() }
 	$done++;
 	plog('NOTIFY', "Build package $srpm [$done/$to_compile]");
@@ -849,7 +835,7 @@ do {
 	    $run{status}{$srpm} = 'missing';
 	    dump_cache_par(\%run);
 	    dump_status($local_spool, \%run);
-	    next;
+	    return;
 	} 
 	# FIXME unfortunately urpmi stalls quite often
 	my $retry = 0;
@@ -864,14 +850,14 @@ retry:
 	
 	if (!$run{use_old_chroot}) {
 	    $chroot_tmp = create_temp_chroot(\%run, $config,
-		$chroot_tmp, $chroot_ref) or next; 
+		$chroot_tmp, $chroot_ref) or return;
 	}
 
 	if (!$urpmi->urpmi_command($chroot_tmp)) {
 	    plog('DEBUG', "Creating chroot failed.\nCommand was: $chroot_tmp");
-	    next;
+	    return;
 	}
-	$srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
+	$srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or return;
 	my ($maintainer, $cc);
 	if (!$run{warn}) {
 	    ($maintainer) = get_maint(\%run, $srpm);
@@ -885,7 +871,7 @@ retry:
 	#($maintainer, $cc) = ($config->{admin},'');
 
 	plog('DEBUG', "creating user $luser in chroot");
-	add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or next;
+	add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or return;
 
 	my $old_srpm = $srpm;
 	my ($ret, $spec);
@@ -908,7 +894,7 @@ retry:
 	    $run{status}{$srpm} = 'recreate_srpm_failure';
 	    dump_cache_par(\%run);
 	    dump_status($local_spool, \%run);
-	    next;
+	    return;
 	} 
 
 	(my $log_dirname = $srpm) =~ s/.*:(.*)\.src.rpm/$1/;
@@ -928,7 +914,7 @@ retry:
 	my $ok = $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'install_deps', "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", { maintainer => $maintainer }, "$path_srpm/$srpm");
 	if (!$ok) {
 	    $run{status}{$srpm} ||= 'install_deps_failure';
-	    next;
+	    return;
 	}
 
 	# try to workarround the rpm -qa db4 error(2) from dbcursor->c_get:
@@ -1009,7 +995,7 @@ retry:
 	    # 20060615
 	    dump_cache_par(\%run);
 	    dump_status($local_spool, \%run);
-	    next;
+	    return;
 	}
 
 	# do some cleaning if the compilation is successful
@@ -1019,7 +1005,7 @@ retry:
 	opendir my $binfh, "$chroot_tmp/home/$luser/rpmbuild/RPMS/";
 	my @packages;
 	foreach my $bindir (readdir $binfh) {
-	    -d "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir" or next;
+	    -d "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir" or return;
 	    opendir my $rpmfh, "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir";
 	    push @packages, map { "$chroot_tmp/home/$luser/rpmbuild/RPMS/$bindir/$_" } grep { !/src\.rpm$/ && /\.rpm$/ } readdir $rpmfh;
 	}
@@ -1028,7 +1014,7 @@ retry:
 	# segfaulting when trying to install packages
 
 	if ($config->{check_binary_file}) {
-	    $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", { maintainer => $maintainer } ,@packages) or next;
+	    $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", { maintainer => $maintainer } ,@packages) or return;
 	} else  {
 	    my $successfile = "$local_spool/log/$srpm/binary_test_$srpm-1.log";
 	    open my $f, ">$successfile";
@@ -1066,6 +1052,63 @@ retry:
 	dump_status($local_spool, \%run);
 	# dump_cache each time so that concurrent process can get updated
 	dump_cache_par(\%run) if $run{concurrent_run};
+    return 1;
+}
+
+my $rebuild;
+$run{group} = 0 if @{$run{todo}} == 1;
+if ($run{group}) { 
+    $rebuild = 1;
+    $urpmi->set_local_media($local_spool);
+    $urpmi->order_packages(\%provides, $luser)
+	or die "FATAL $program_name: could not order packages";
+}	
+#
+# The build loop
+#
+my $prev_done = $done;
+do { 
+    $rebuild = 0;
+    $done = $prev_done;
+    my $i;
+    my %children;
+    for ($i; $i < @{$run{todo}}; $i++) {
+	my ($dir, $srpm, $status) = @{$run{todo}[$i]};
+	if ($run{parallel}) {
+	    my $pid;
+	    # First cleanup all the finished ones
+	    do {
+		$pid = waitpid(-1, WNOHANG);
+		if ($pid > 0) {
+		    plog('INFO', "Child $pid has exited");
+		    delete $children{$pid};
+	        }
+	    } while $pid > 0;
+	    while (scalar keys %children >= $run{parallel}) {
+		plog('INFO', "Too many children, waiting to fork more");
+		$pid = waitpid(-1, 0);
+		delete $children{$pid} if $pid > 0;
+	    }
+	    my $pid = fork;
+	    if ($pid) { #parent
+		$children{$pid} = 1;
+	    } elsif ($pid == 0) { #child
+		$chroot_tmp .= "_" . int($i);
+		rebuild_one($dir, $srpm, $status);
+		exit;
+	    } else {
+		die "could not fork";
+	    }
+	} else {
+	    rebuild_one($dir, $srpm, $status);
+	}
+    }
+    if ($run{parallel}) {
+	foreach my $pid (keys %children) {
+	    plog('INFO', "Waiting for process $pid to exit");
+	    waitpid($pid, 0);
+	    delete $children{$pid};
+	}
     }
     if ($run{group}) {
 	my $i;