- begin to add group option to compile several srpm as a set

- fix unionfs dir cleaning
- create a function to install srpm
- retry to install packages dependencies when urpmi failed, quite often on x86_64 the rpm database get corrupted and rpm does not find correctly which packages are installed
- correclty check if a dir is mounted or not
- retry several time to umount directory (they can be mounted multiple times)

1 files changed, 429 insertions, 210 deletions

diff --git a/iurt2 b/iurt2
index 61bef29..0b6c52b 100755
--- a/iurt2
+++ b/iurt2
@@ -50,7 +50,7 @@ $run{todo} = [ ];
     #
     [ "", "$program_name", 0, "[--cache] [--chrooted-urpmi <media prefix>] [--concurrent-run] [--config foo value] [--warn] [--verbose integer]
             [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}]
- 	    [--use-system-distrib] [--dir] [--help foo?] [--log filename]  [--unionfs]
+ 	    [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs]
  	    [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename]  [--unionfs]
  	    {--config_help |
 	    --chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } |
@@ -102,6 +102,9 @@ $run{todo} = [ ];
     [ "", "debug", 0, "", 
     "Activate debug mode", 
     sub { $run{debug} = 1 }, "Activating debug mode" ],
+    [ "g", "group", 0, "", 
+    "Activate group mode, packages will be compiled as a global set, not as individual packages", 
+    sub { $run{group} = 1 }, "Activating group mode to compile packages as a set" ],
     [ "u", "unionfs", 0, "", 
     "Activate unionfs mode", 
     sub { $run{unionfs} = 1 }, "Activating unionfs mode" ],
@@ -134,7 +137,7 @@ $run{todo} = [ ];
 	    my $hdr = rpm2header($_);
 	    if (check_arch($hdr)) {
 		print {$run{LOG}} "iurt: force build for $2 (from $1)\n";
-		push @{$run{todo}}, [ $path, $srpm ]
+		push @{$run{todo}}, [ $path, $srpm, 1 ]
 	    } else {
 		print {$run{LOG}} "ERROR iurt: $_ could not be build on $run{my_arch}, ignored.\n"
 	    }
@@ -221,10 +224,21 @@ my $urpmi_options = "-v --no-verify-rpm --nolock --auto --ignoresize";
 if ($run{use_system_distrib}) {
     $config->{basesystem_media_root} ||= $run{use_system_distrib}
 } elsif ($run{chrooted_urpmi}) {
-    $run{chrooted_media} = "$run{chrooted_urpmi}/$run{distro}/$run{my_arch}"
+    my ($host) = $run{chrooted_urpmi} =~ m,(?:file|http|ftp)://([^/]*),;
+    my ($name,$aliases,$addrtype,$length,@addrs) = gethostbyname $host;
+    my $ip = join '.', unpack('C4',$addrs[0]);
+    $ip =~ /\d+\.\d+\.\d+\.\d+/ or die "FATAL iurt: could not resolve $host ip address";
+    $run{chrooted_urpmi} =~ s/$host/$ip/;
+    $run{chrooted_media} = "$run{chrooted_urpmi}/$run{distro}/$run{my_arch}";
+    print {$run{LOG}} "iurt: using $run{chrooted_media} as insallation media\n" if $run{verbose} > 2
 } else {
     $urpmi_options .= " --use-distrib $config->{repository}/$run{distro}/$run{my_arch}"
 }
+
+if (!$run{chrooted_urpmi} && $run{group}) {
+    die "FATAL iurt: option --chrooted-urpmi is mandatory if --group is selected"
+} 
+
 my %config_usage = ( 
     admin => { desc => 'Mail of the administrator of packages builds', default => '' },
     all_media => { desc => 'List of known media', default => [ 'main', 'contrib' ] },
@@ -291,7 +305,7 @@ my $chroot_name = "chroot_$run{distro_tag}$debug_tag";
 my $chroot = "$config->{local_home}/$chroot_name";
 my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
 if ($run{chroot}) {
-    check_chroot($chroot, $chroot_tar, \%run)
+    check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot"
 }
 
 my $cache;
@@ -395,8 +409,8 @@ if ($df->{per} == 100) {
 }
 
 if ($run{shell}) {
-    my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id);
-    add_local_user($chroot_tmp, \%run, $luser, $run{uid});
+    ($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or die "FATAL iurt: could not create temporary chroot";
+    add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL iurt: could not add local user";
     $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp";
     add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi');
     add_sudoers(\%run, $luser);
@@ -430,7 +444,7 @@ if ($config->{rsync_to} && !$run{no_rsync}) {
 print {$run{LOG}} "iurt: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3;
 if (!dump_rpmmacros("$chroot/home/builder/.rpmmacros")) {
      print "ERROR iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros, trying to rebuild the chroot";
-     check_chroot($chroot, $chroot_tar, \%run);
+     check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot";
      dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros"
 }
 
@@ -440,7 +454,7 @@ my $s = sub {
 	dump_cache(\%run);
 	$Data::Dumper::Indent = 0;
 	$Data::Dumper::Terse = 1;
-	print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n"; 
+	print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n" if $run{verbose} > 5; 
 	print {$run{LOG}} "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n";
     }
     exit 
@@ -449,8 +463,18 @@ $SIG{TERM} = $s;
 $SIG{INT} = $s;
 $run{main} = 1;
 
-foreach (my $i ; $i < @{$run{todo}}; $i++) {
-	my ($dir, $srpm) = @{$run{todo}[$i]};
+my $local_media;
+my $rebuild;
+if ($run{group}) { 
+    $rebuild = 1;
+    order_packages(\%run) or die "FATAL iurt: could not order packages";
+    $local_media = "$local_spool/$run{run}/"
+}
+
+do { 
+    foreach (my $i ; $i < @{$run{todo}}; $i++) {
+	my ($dir, $srpm, $status) = @{$run{todo}[$i]};
+	$status or next;
 	$done{$srpm} and next;
 	$done{$srpm} = 1;
 	check_version($srpm) or next;
@@ -460,20 +484,37 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 	print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n";
 	# FIXME unfortunately urpmi stalls quite often
 	my $retry = 0;
-retry:
-        if (!$run{chrooted_urpmi}) {
+	retry:
+	if (!$run{chrooted_urpmi}) {
 	    my $match = "urpmi $urpmi_options --root $chroot_tmp";
 	    if (!clean_process($match, $run{verbose})) {
-	        dump_cache(\%run);
-	        die "FATAL iurt: Could not have urpmi working !"
+		dump_cache(\%run);
+		die "FATAL iurt: Could not have urpmi working !"
 	    }
 	}
-	my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm);
+	($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm) or next;
 
 	if ($run{chrooted_urpmi}) { 
 	    $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp ";
 	    add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi');
 	    add_media(\%run, $config, $chroot_tmp);
+	    if (-d "$local_media/hdlist.cz") {
+		mkdir("$chroot_tmp/iurt_media/");
+		opendir my $dir, $local_media;
+		my $next;
+		foreach my $f (readdir $dir) {
+		    $f =~ /(\.rpm|^hdlist.cz)$/ or next;
+		    if (!link "$local_media/$f", "$chroot_tmp/iurt_media") {
+			if (!copy "$local_media/$f", "$chroot_tmp/iurt_media") {
+			    print {$run{LOG}} "ERROR iurt: could not copy file $local_media/$f to $chroot_tmp/iurt_media";
+			    $next = 1;
+			    last
+			}
+		    }
+		}
+		next if $next;
+		add_local_media(\%run, $config, $chroot_tmp, "/iurt_media")
+	    }
 	    $run{urpmi_command} = "chroot $chroot_tmp urpmi $urpmi_options "
 	} else {
 	    $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp"
@@ -491,71 +532,36 @@ retry:
 	}
 	#($maintainer, $cc) = ($config->{admin},'');
 
-	print {$run{LOG}} "iurt: adding local user to the chroot...\n" if $run{verbose};
-	add_local_user($chroot_tmp, \%run, $luser, $run{uid});
+	print {$run{LOG}} "iurt: adding local user $luser into $chroot_tmp...\n" if $run{verbose};
+	add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or next;
 
-	# recreate a new srpm for buildarch condition in the spec file
-	print {$run{LOG}} "iurt: copying $srpm to $chroot_tmp\n" if $run{verbose} > 1;
-	perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/", 
-	        \%run, $config,
-		mail => $config->{admin}, 
-		error => "[REBUILD] cannot copy $srpm to $chroot_tmp", 
-		debug_mail => $run{debug},
-		hash => "copy_$srpm") or next;
-
-	my %opt = ( mail => $config->{admin}, 
-		error => "[REBUILD] cannot install $srpm in $chroot_tmp", 
-		debug_mail => $run{debug},
-		hash => "install_$srpm",
-		retry => $retry,
-		callback => sub { 
-			my ($opt, $output) = @_;
-			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
-			if ($output =~ /user $luser does not exist/) {
-			    print {$run{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run{verbose} > 1;
-			    $opt->{error} = "[CHROOT] chroot is corrupted";
-			    $opt->{retry} = 1 if !$opt->{retry};
-			    return 
-			}
-			1
-		    }
-		);
-	print {$run{LOG}} "iurt: recreating src.rpm...\n" if $run{verbose};
-	if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"}, 
-		\%run, $config, %opt)) {
-	        print {$run{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run{debug};
-		if ($opt{retry}) {
-		    $retry = 1;
-		    check_chroot($chroot, $chroot_tar, \%run);
-		    goto retry
-		}
-		die;
-		next
-	    }
-	    # This should not be necessairy any more if urpmi *.spec works, but it doesn't
-	    #
-	perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm --nodeps -bs /home/$luser/rpm/SPECS/*.spec"}, 
-		\%run, $config,
-		mail => $config->{admin}, 
-		error => "[REBUILD] cannot create $srpm in $chroot_tmp", 
-		debug_mail => $run{debug},
-		hash => "create_$srpm") or next;
-	print {$run{LOG}} "iurt: installing build dependencies of $srpm...\n" if $run{verbose};
+	my $ret = recreate_srpm(\%run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry);
+	if ($ret == -1) {
+	    $retry = 1;
+	    goto retry
+	} elsif (!$ret) {
+	    next
+	} 
 	
+	print {$run{LOG}} "iurt: installing build dependencies of $srpm...\n" if $run{verbose};
 	my $wait_urpmi = sub { 
-		    print {$run{LOG}} "WARNING iurt: urpmi database locked, waiting...\n" if $run{debug};
-		    sleep 30; 
-		    $wait_limit++; 
-		    if ($wait_limit > 8) { 
-			$wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) 
-		    } };
+	    print {$run{LOG}} "WARNING iurt: urpmi database locked, waiting...\n" if $run{debug};
+	    sleep 30; 
+	    $wait_limit++; 
+	    if ($wait_limit > 8) { 
+		$wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) 
+	    } };
 	my $path_srpm =  $run{chrooted_urpmi} ? "/home/$luser/rpm/SRPMS/" : "$chroot_tmp/home/$luser/rpm/SRPMS/";
-	if (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm", 
+	# 
+	# on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore 
+	# installed packages, retrying several time to be sure something is really broken
+	my $install_deps_retry = 5;
+	while (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm", 
 		\%run, $config,
 		mail => $config->{admin},
 		error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}",  
 		hash => "install_deps_$srpm", 
-		timeout => 600, 
+		timeout => 900, 
 		srpm => $srpm,
 		freq => 1,
 		#cc => $cc, 
@@ -565,44 +571,49 @@ retry:
 		wait_regexp => 'database locked',
 		wait_callback => $wait_urpmi,
 		log => "$local_spool/log/$srpm/",
+		reopen_log => 1,
 		callback => sub { 
-			my ($opt, $output) = @_;
-			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
-			my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
-			# as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
-			@missing_deps or return 1;
-			while (my $missing_deps = shift @missing_deps) {
-			    my $version = shift @missing_deps;
-			    my $version2 = shift @missing_deps;
-			    $version ||= $version2 || 0;
-			    my $p = $pack_provide{$missing_deps} || $missing_deps;
-			    my $other_maint = `rpmmon -p "$p"`;
-			    chomp $other_maint;
-			    print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
-			    if ($other_maint && $other_maint ne 'NOT_FOUND') {
-				$opt->{mail} = "$maintainer, $other_maint";
-				$opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
-			    }
-			    # remember what is needed, and do not try to recompile until it is available
-			    push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
+		    my ($opt, $output) = @_;
+		    print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
+# 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error).	    
+# my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g;
+		    my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
+		    # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
+		    @missing_deps or return 1;
+		    while (my $missing_deps = shift @missing_deps) {
+			my $version = shift @missing_deps;
+			my $version2 = shift @missing_deps;
+			$version ||= $version2 || 0;
+			my $p = $pack_provide{$missing_deps} || $missing_deps;
+			my $other_maint = `rpmmon -p "$p"`;
+			chomp $other_maint;
+			print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
+			if ($other_maint && $other_maint ne 'NOT_FOUND') {
+			    $opt->{mail} = "$maintainer, $other_maint";
+			    $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
 			}
-			1
+			# remember what is needed, and do not try to recompile until it is available
+			push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
+		    }
+		    0
 		},
-	)) { 
-	    $run{status}{$srpm} = 'install_deps_failure';
-	    next
+	    )) { 
+	    $install_deps_retry--;
+	    foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
+	    if (!$install_deps_retry || $cache->{needed}{$srpm}) {
+		$run{status}{$srpm} = 'install_deps_failure';
+		next
+	    }
 	}
 	# try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory
 	# running rpm -qa several time seems to fix the problem
-	foreach (1 .. 3) {
-	    system("sudo chroot $chroot_tmp rpm -qa &> /dev/null")
-	}
+	foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
 	perform_command("sudo chroot $chroot_tmp rpm -qa", 
-		\%run, $config,
-		hash => "rpm_qa_$srpm", 
-		timeout => 60, 
-		debug_mail => $run{debug},
-		log => "$local_spool/log/$srpm/"); # or next; As this failed quite often, do not stop
+	    \%run, $config,
+	    hash => "rpm_qa_$srpm", 
+	    timeout => 60, 
+	    debug_mail => $run{debug},
+	    log => "$local_spool/log/$srpm/"); # or next; As this failed quite often, do not stop
 	print {$run{LOG}} "Compiling $srpm\n" if $run{verbose}; 
 	my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm";
 	if ($run{stop}) {
@@ -611,84 +622,85 @@ retry:
 	    $command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec"
 	}
 	if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, 
-			\%run, $config,
-			mail => $maintainer, 
-			error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", 
-			hash => "build_$srpm", 
-			timeout => 18000, 
-			srpm => $srpm,
-			debug_mail => $run{debug},
-			cc => $cc, 
-			log => "$local_spool/log/$srpm/", 
-			error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
-			callback => sub { 
-			    my ($opt, $output) = @_;
-			    if ($run{stop}) {
-				print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp (pid $$)\n";
-				# exec does not work because it seems stdin and out are shared between children
-				system("sudo chroot $chroot_tmp /bin/su $luser -c bash");
-				exit
-			    }
-			    print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug};
-			    if ($unionfs_tmp && $output =~ /no space left on device/i) {
-				print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
-				$cache->{no_unionfs}{$srpm} = 1;
-				return 1
-			    } elsif ($unionfs_tmp && $output =~ m,$home,) {
-				print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n";
-				$cache->{no_unionfs}{$srpm} = 1;
-				return 1
-			    }
-			    1
-			}, 
-			freq => 1)) {
-		    # FIXME
-		    # The simple algo used here is : 
-		    #  try to compile it with unionfs, if it runs out of space, compile it without the next time
-		    #
-		    #  This could be improved in keeping this srpm name for future version, but if we compile it 
-		    #  on a new machine with more ram, or if next version compiles just fine with unionfs, we will
-		    #  loose the unionfs advantage. 
-		    #
-		    #  Maybe the right thing to do would be to first try to increase the tmpfs size (more than 50 % of the 
-		    #  physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs
-		    #  plus). Or to keep the faulty package a unionfs exception for some time, to save some more extra builds.
-		    # 
-		    if (!$unionfs_tmp) {
-			$run{status}{$srpm} = 'build_failure';
-			$cache->{failure}{$srpm} = 1
-		    } elsif (!$cache->{no_unionfs}{$srpm}) {
-			$run{status}{$srpm} = 'build_failure';
-			$cache->{failure}{$srpm} = 1
-		    } else {
-			goto retry
+		\%run, $config,
+		mail => $maintainer, 
+		error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", 
+		hash => "build_$srpm", 
+		timeout => 18000, 
+		srpm => $srpm,
+		debug_mail => $run{debug},
+		cc => $cc, 
+		log => "$local_spool/log/$srpm/", 
+		error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
+		callback => sub { 
+		    my ($opt, $output) = @_;
+		    if ($run{stop}) {
+			print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp (pid $$)\n";
+			# exec does not work because it seems stdin and out are shared between children
+			system("sudo chroot $chroot_tmp /bin/su $luser -c bash");
+			exit
 		    }
-		    if (!glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") {
-			next
+		    print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug};
+		    if ($unionfs_tmp && $output =~ /no space left on device/i) {
+			print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
+			$cache->{no_unionfs}{$srpm} = 1;
+			return 1
+		    } elsif ($unionfs_tmp && $output =~ m,$home,) {
+			print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n";
+			$cache->{no_unionfs}{$srpm} = 1;
+			return 1
 		    }
+		    1
+		}, 
+		freq => 1)) {
+	    # FIXME
+	    # The simple algo used here is : 
+	    #  try to compile it with unionfs, if it runs out of space, compile it without the next time
+	    #
+	    #  This could be improved in keeping this srpm name for future version, but if we compile it 
+	    #  on a new machine with more ram, or if next version compiles just fine with unionfs, we will
+	    #  loose the unionfs advantage. 
+	    #
+	    #  Maybe the right thing to do would be to first try to increase the tmpfs size (more than 50 % of the 
+	    #  physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs
+	    #  plus). Or to keep the faulty package a unionfs exception for some time, to save some more extra builds.
+	    # 
+	    if (!glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") {
+		if ($unionfs_tmp && $cache->{no_unionfs}{$srpm}) {
+		    goto retry
+		}
+		$cache->{failure}{$srpm} = 1;
+		$run{status}{$srpm} = 'build_failure';
+		next
+	    }
 	}
 	# do some cleaning if the compilation is successful
 	delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm};
 	if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", 
-			\%run, $config,
-			mail => $maintainer, 
-			error => "[REBUILD] binaries packages generated from $srpm do not install correctly", 
-			hash => "binary_test_$srpm", 
-			srpm => $srpm,
-			timeout => 600, 
-			debug_mail => $run{debug},
-			freq => 1, 
-			wait_regexp => 'database locked',
-			wait_callback => $wait_urpmi,
-error_regexp => 'unable to access',
-			log => "$local_spool/log/$srpm/")) {
-		$cache->{failure}{$srpm} = 1;
-		$run{status}{$srpm} = 'binary_test_failure';
-		next
+		\%run, $config,
+		mail => $maintainer, 
+		error => "[REBUILD] binaries packages generated from $srpm do not install correctly", 
+		hash => "binary_test_$srpm", 
+		srpm => $srpm,
+		timeout => 600, 
+		debug_mail => $run{debug},
+		freq => 1, 
+		wait_regexp => 'database locked',
+		wait_callback => $wait_urpmi,
+		error_regexp => 'unable to access',
+		log => "$local_spool/log/$srpm/")) {
+	    $cache->{failure}{$srpm} = 1;
+	    $run{status}{$srpm} = 'binary_test_failure';
+	    next
 	}
 	if ($run{debug}) {
 	    print {$run{LOG}} "iurt: debug mode, skip other packages\n";
 	    exit
+	} elsif ($run{group}) {
+	    print {$run{LOG}} "iurt: group mode, keep packages for local media\n";
+	    $run{done}{$srpm} = $done;
+	    system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_media ($!)\n";
+	    system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_media ($!)\n";
 	} else {
 	    print {$run{LOG}} "iurt: build successful, copying packages to $local_spool.\n";
 	    $run{status}{$srpm} = 'ok';
@@ -699,8 +711,12 @@ error_regexp => 'unable to access',
 	    process_queue($config, \%run, \@wrong_rpm, 1)
 	}
 	# dymp_cache each time so that concurrent process can get updated
-	dump_cache(\%run) if $run{concurrent_run}
-}
+	dump_cache(\%run) if $run{concurrent_run};
+    }
+    if ($run{group}) {
+	$rebuild = 1 if order_packages(\%run)
+    }
+} while ($rebuild);
 
 print {$run{LOG}} "iurt: reprocess generated packages queue\n" if $run{verbose};
 process_queue($config, \%run, \@wrong_rpm);
@@ -769,7 +785,7 @@ sub clean_all_unionfs {
     }
     foreach (readdir $dir) {
 	/unionfs\.((?:0\.)?\d+)\.(\d+)$/ or next;
-	clean_unionfs($unionfs_dir, $1, $2);
+	clean_unionfs($unionfs_dir, \%run, $1, $2);
     }
     closedir $dir
 }
@@ -777,7 +793,11 @@ sub clean_all_unionfs {
 sub clean_all_chroot_tmp {
     my ($chroot_dir, $prefix) = @_;
     print {$run{LOG}} "Cleaning old chroot remaining dir in $chroot_dir\n" if $run{verbose};
-    opendir my $dir, $chroot_dir or die "FATAL iurt: could not open $chroot_dir ($!)";
+    my $dir;
+    if (!opendir $dir, $chroot_dir) { 
+	print "ERROR iurt: could not open $chroot_dir ($!)";
+	return
+    }
     foreach (readdir $dir) {
 	/$prefix.*$/ or next;
 	clean_chroot_tmp($chroot_dir, $_);
@@ -837,31 +857,41 @@ sub clean {
 }
 
 sub clean_unionfs {
-    my ($unionfs_dir, $run, $union_id) = @_;
-    if (system("sudo umount $unionfs_dir/unionfs.$run.$union_id/proc &>/dev/null") && $run{verbose} > 1) { 
-	print {$run{LOG}} "ERROR iurt: could not umount /proc in $unionfs_dir/unionfs.$run.$union_id\n" 
-    }
-    print {$run{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run{verbose} > 1;
+    my ($unionfs_dir, $run, $r, $union_id) = @_;
+    -d "$unionfs_dir/unionfs.$r.$union_id" or return $union_id;
+    print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run->{verbose} > 1;
     my $nok = 1;
+    my $proc = "$unionfs_dir/unionfs.$r.$union_id/proc";
     while ($nok) {
+	$nok = 0;
+	if (-d $proc && check_mounted($proc, 'proc')) {
+	    print {$run->{LOG}} "iurt iurt: umounting $proc\n" if $run->{verbose} > 2;
+	    if (system("sudo umount $proc &>/dev/null")) { 
+		print {$run->{LOG}} "ERROR iurt: could not umount $proc\n";
+		return $union_id + 1
+	    }
+	}
 	foreach my $t ("unionfs",'tmpfs') {
 	    # unfortunately quite oftem the unionfs is busy and could not be unmounted
-	    my $d = "$unionfs_dir/$t.$run.$union_id";
+	    my $d = "$unionfs_dir/$t.$r.$union_id";
 	    my $last;
-	    while (check_mounted($d, $t)) {
+	    if (-d $d && check_mounted($d, $t)) {
+		$nok = 1;
 		system("sudo /sbin/fuser -k $d &> /dev/null");
-		print {$run{LOG}} "iurt: umounting $d\n" if $run{verbose} > 2;
+		print {$run->{LOG}} "iurt: umounting $d\n" if $run->{verbose} > 2;
 		if (system(qq{sudo umount $d &> /dev/null})) {
-		    print {$run{LOG}} "WARNING iurt: could not umount $d ($!)\n" if $run{verbose} > 1;
-		    $union_id++;
-		    $last = 1;
-		    last
+		    print {$run->{LOG}} "WARNING iurt: could not umount $d ($!)\n" if $run->{verbose} > 1;
+		    return $union_id + 1;
 		}
 	    }
-	    $last and last;
-	    print {$run{LOG}} "iurt: removing $d\n" if $run{verbose} > 1;
-	    system(qq{sudo rm -rf $d});
-	    $nok = 0
+	}
+    }
+    foreach my $t ("unionfs",'tmpfs') {
+	my $d = "$unionfs_dir/$t.$r.$union_id";
+	print {$run->{LOG}} "iurt: removing $d\n" if $run->{verbose} > 1;
+	if (system(qq{sudo rmdir $d})) {
+	    print {$run->{LOG}} "ERROR iurt: removing $d failed ($!)\n";
+	    return $union_id + 1
 	}
     }
     $union_id
@@ -881,7 +911,11 @@ sub clean_chroot_tmp {
 
 sub check_mounted {
     my ($mount_point, $type) = @_;
-    open my $mount, '/proc/mounts' or die 'FATAL iurt: could not open /proc/mounts';
+    my $mount;
+    if (!open $mount, '/proc/mounts') {
+	print 'ERROR iurt: could not open /proc/mounts';
+	return
+    }
     $mount_point =~ s,//+,/,g;
     while (<$mount>) {
 	return 1 if /^\w+ $mount_point $type /
@@ -995,7 +1029,7 @@ sub dump_cache {
 	    print {$run{LOG}} "ERROR iurt: manual file lock exist, do not save the cache\n";
 	} else {
 	    open my $lock, ">$filename.lock";
-	    print $lock, $$;
+	    print $lock $$;
 	    close $lock;
 	    unlink $filename;
 	    link "$filename.tmp.$daydate", $filename;
@@ -1106,7 +1140,11 @@ sub perform_command {
 	    print {$run{LOG}} "$command\n" if $run{verbose} > 2;
 	    if ($opt{log}) {
 		#$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
-		system("$command &> $logfile");
+		if ($opt{reopen_log}) {
+		    system("$command 2>&1 >> $logfile");
+		} else {
+		    system("$command >& $logfile");
+		}
 	    } else {
 		$output = `$command 2>&1`;
 	    }
@@ -1162,7 +1200,13 @@ sub perform_command {
 	if ($opt{log} && $config->{log_url}) {
 	    $comment = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$comment|
 	}
-	my $out = $output if length $output < 10000;
+	my $out;
+	if (length $output < 10000) {
+	    $out = $output
+	} else { 
+	    $out = "Message too big, see http link for details\n"
+	}
+
 	if ($opt{mail} && $config->{sendmail} && !$config->{no_mail}{$opt{mail}}) {
 	    if (! ($cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq})) {
 		my $cc = join ',', grep { !$config->{no_mail}{$_} } split ',', $opt{cc};
@@ -1199,7 +1243,10 @@ sub check_chroot {
     my ($chroot, $chroot_tar, $run) = @_;
     print {$run{LOG}} "iurt: checking basesystem tar\n" if $run{verbose};
     system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot" &> /dev/null});
-    clean_chroot($chroot, $run, 1) or die "FATAL iurt: Could no prepare initial chroot";
+    if (!clean_chroot($chroot, $run, 1)) {
+	print "ERROR iurt: Could no prepare initial chroot";
+	return
+    }
     perform_command("sudo $config->{install_chroot_binary} $config->{basesystem_media_root} $config->{basesystem_media_root}/media/$config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build rpm-mandriva-setup-build", 
 	$run, $config,
 	mail => $config->{admin},
@@ -1400,7 +1447,7 @@ sub search_packages {
 		    $maint{$srpm} = $mail;
 		    print "iurt: will try to compile $srpm\n";
 		    $to_compile++;
-		    push @{$run->{todo}}, [ $dir , $srpm ]
+		    push @{$run->{todo}}, [ $dir , $srpm, 1 ]
 		}
 		foreach my $arch (@{$config->{supported_arch}}) {
 		    $ok &&= $cache->{queue}{$srpm}
@@ -1431,16 +1478,20 @@ sub add_local_user {
     # change the builder user to the local user id
     # FIXME it seems that unionfs does not handle well the change of the uid of files
     # if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) {
-    if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) && system(qq|sudo chroot $chroot_tmp id $luser|)) {
-	print {$run->{LOG}} "ERROR: setting userid to builder user in chroot failed, trying to check the chroot\n";
+    if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) || system(qq|sudo chroot $chroot_tmp id $luser|)) {
+	print {$run->{LOG}} "ERROR: setting userid $uid to $luser in $chroot_tmp failed, trying to check the chroot\n";
 	check_chroot($chroot, $chroot_tar, $run);
+	return
     }
     if (system(qq|sudo chroot $chroot_tmp cp -R /home/builder /home/$luser|)) {
-	die "FATAL iurt: could not initialized $luser directory\n";
+	print "ERROR iurt: could not initialized $luser directory\n";
+	return
     }
     if (system(qq|sudo chroot $chroot_tmp chown -R $uid /home/$luser|)) {
-	die "FATAL iurt: could not initialized $luser directory\n";
+	die "ERROR iurt: could not initialized $luser directory\n"; 
+	return
     }
+    1
 }
  
 sub create_temp_chroot {
@@ -1455,14 +1506,20 @@ sub create_temp_chroot {
 	my $tmpfs;
 
 	# we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database)
-	$union_id = clean_unionfs($unionfs_dir, $run->{run}, $union_id);
+	$union_id = clean_unionfs($unionfs_dir, $run, $run->{run}, $union_id);
 	$tmpfs = "$unionfs_dir/tmpfs.$run->{run}.$union_id";
 	$chroot_tmp = "$unionfs_dir/unionfs.$run->{run}.$union_id";
 	if (!-d $tmpfs) {
-	    mkdir $tmpfs or die "FATAL iurt: Could not create $tmpfs ($!)"
+	    if (!mkdir $tmpfs) {
+		print "ERROR iurt: Could not create $tmpfs ($!)";
+		return
+	    }
 	}
 	if (! -d $chroot_tmp) {
-	    mkdir $chroot_tmp or die "FATAL iurt: Could not create $chroot_tmp ($!)"
+	    if (!mkdir $chroot_tmp) {
+		print "ERROR iurt: Could not create $chroot_tmp ($!)";
+		return
+	    }
 	}
 	if ($cache->{no_unionfs}{$srpm}) {
 	    $unionfs_tmp = 0;
@@ -1471,15 +1528,24 @@ sub create_temp_chroot {
 	    # if the previous package has been built without unionfs, chroot need to be cleaned
 	    clean_chroot($chroot_tmp, $run) if !$unionfs_tmp;
 	    $unionfs_tmp = 1;
-	    system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null}) and die "FATAL iurt: could not mount $tmpfs ($!)";
-	    system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run->{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|) and die "FATAL iurt: could not mount $tmpfs and $home/chroot_$run->{distro_tag}$debug_tag with unionfs ($!)";
-	    system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null") and die "FATAL iurt: could not mount /proc in the chroot $chroot_tmp.";
+	    if (system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null})) {
+		print "ERROR iurt: could not mount $tmpfs ($!)"; 
+		return
+	    }
+	    if (system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run->{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|)) {
+		print "ERROR iurt: could not mount $tmpfs and $home/chroot_$run->{distro_tag}$debug_tag with unionfs ($!)";
+		return
+	    }
+	    if (system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null")) {
+		print "ERROR iurt: could not mount /proc in the chroot $chroot_tmp.";
+		return
+	    }
 	}
     } else {
 	print {$run->{LOG}} "iurt: installing a new chroot in $chroot_tmp\n" if $run->{verbose} > 1;
 	clean_chroot($chroot_tmp, $run)
     }
-    $unionfs_tmp, $chroot_tmp
+    $union_id, $unionfs_tmp, $chroot_tmp
 }
 
 sub add_packages {
@@ -1540,11 +1606,164 @@ sub remove_chroot {
 sub add_media {
     my ($run, $config, $chroot) = @_;
     print {$run->{LOG}} "iurt: adding media $run{chrooted_media} in chroot $chroot\n";
-    perform_command("sudo chroot $chroot_tmp urpmi.addmedia --distrib $run{chrooted_media}", 
+    if (!perform_command("sudo chroot $chroot urpmi.addmedia --distrib $run{chrooted_media}", 
+		$run, $config,
+		mail => $config->{admin},
+		timeout => 120, 
+		freq => 1,
+		retry => 5,
+		debug_mail => $run->{debug})) {
+	}
+    check_media_added($chroot, 'Main') or die "FATAL iurt: could not add media into the chroot"
+}
+
+sub add_local_media {
+    my ($run, $config, $chroot, $local_media) = @_;
+    print {$run->{LOG}} "iurt: adding media $local_media in chroot $chroot\n";
+    if (perform_command("sudo chroot $chroot urpmi.addmedia iurt_group file://$local_media", 
 		$run, $config,
 		mail => $config->{admin},
 		timeout => 120, 
 		freq => 1,
 		retry => 5,
-		debug_mail => $run->{debug})
+		debug_mail => $run->{debug})) {
+	}
+    check_media_added($chroot, 'iurt_group') or die "FATAL iurt: could not add local media into the chroot"
+}
+
+sub check_media_added {
+    my ($chroot, $media) = @_;
+    my $medias = `sudo chroot $chroot urpmi.removemedia 2>&1`;
+    print "MEDIA $medias\n";
+    $medias =~ /one of.* $media/m
+}
+
+sub get_local_provides {
+    my ($run, $local_media) = @_;
+    opendir my $dir, $local_media;
+    my $urpm = new URPM;
+    foreach my $d (readdir $dir) {
+	$d =~ /\.rpm$/ or next;
+	my $id = $urpm->parse_rpm("$local_media/$d");
+	my $pkg = $urpm->{depslist}[$id];
+        foreach ($pkg->provides, $pkg->files) {
+	    print {$run->{LOG}} "iurt: adding $_ as provides of $d\n" if $run->{verbose} > 2;
+	    $run{local_provides}{$_} = $d
+	}
+    }
+}
+
+sub recreate_srpm {
+    my ($run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry) = @_;
+# recreate a new srpm for buildarch condition in the spec file
+    print {$run->{LOG}} "iurt: copying $srpm to $chroot_tmp\n" if $run->{verbose} > 1;
+    perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/", 
+	\%run, $config,
+	mail => $config->{admin}, 
+	error => "[REBUILD] cannot copy $srpm to $chroot_tmp", 
+	debug_mail => $run->{debug},
+	hash => "copy_$srpm") or return;
+
+    my %opt = ( mail => $config->{admin}, 
+	error => "[REBUILD] cannot install $srpm in $chroot_tmp", 
+	debug_mail => $run->{debug},
+	hash => "install_$srpm",
+	retry => $retry,
+	callback => sub { 
+	    my ($opt, $output) = @_;
+	    print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug};
+	    if ($output =~ /warning: group .* does not exist - using root/i) {
+		return 1
+	    } elsif ($output =~ /user $luser does not exist/) {
+		print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1;
+		$opt->{error} = "[CHROOT] chroot is corrupted";
+		$opt->{retry} = 1 if !$opt->{retry};
+		return 
+	    }
+	});
+    print {$run->{LOG}} "iurt: recreating src.rpm...\n" if $run->{verbose};
+    if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"}, 
+	    \%run, $config, %opt)) {
+	print {$run->{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run->{debug};
+	if ($opt{retry}) {
+	    check_chroot($chroot, $chroot_tar, \%run) or return;
+	    return -1
+	}
+	return
+    }
+    # 20060515 This should not be necessairy any more if urpmi *.spec works, but it doesn't
+    #
+    perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm --nodeps -bs /home/$luser/rpm/SPECS/*.spec"}, 
+	\%run, $config,
+	mail => $config->{admin}, 
+	error => "[REBUILD] cannot create $srpm in $chroot_tmp", 
+	debug_mail => $run->{debug},
+	hash => "create_$srpm")
+}
+
+sub get_build_requires {
+    my ($run, $config) = @_;
+    $run{todo_requires} = {};
+    my ($union_id, $unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or return;
+    
+    my $urpm = new URPM;
+    foreach my $p (@{$run->{todo}}) {
+	my ($dir, $srpm, $s) = @$p;
+	recreate_srpm($run, $config, $chroot_tmp, $dir, $srpm, 'iurt') or return;
+	$s or next;
+	my $id = $urpm->parse_rpm("$dir/$srpm");
+	my $pkg = $urpm->{depslist}[$id];
+        foreach ($pkg->requires) {
+	    print {$run->{LOG}} "iurt: adding $_ as requires of $srpm\n" if $run->{verbose} > 2;
+	    $run{todo_requires}{$_} = $srpm
+	}
+    }
+}
+
+sub order_packages {
+    my ($run, $config, $provides, $local_media) = @_;
+    my @packages = @{$run{todo}};
+    my $move;
+    get_local_provides($run, $local_media) or return;
+    if (!$run{todo_requires}) {
+	get_build_requires($run, $config) or return
+    }
+    my %visit;
+    my %status;
+    do { 
+	$move = 0;
+	foreach my $p (@packages) {
+	    my ($dir, $rpm, $status) = @$p;
+	    defined $status{$rpm} && $status{$rpm} == 0 and next;
+	    print {$run->{LOG}} "iurt: checking packages $rpm\n";
+	    foreach my $r (@{$run->{todo_requires}{$rpm}}) {
+		print {$run->{LOG}} "iurt: checking requires $r\n";
+		if (!$run->{local_provides}{$r}) { 
+		    if ($provides->{$r}) {
+			$status = 1
+		    } else {
+			$status = 0
+		    }
+		} elsif ($visit{$rpm}{$r}) {
+		    # to evit loops
+		    $status = 0
+		} elsif ($run->{done}{$rpm} && $run->{done}{$provides->{$r}}) {
+		    if ($run->{done}{$rpm} < $run->{done}{$provides->{$r}}) {
+			$move = 1;
+			$status = $status{$provides->{$r}} + 1 
+		    } else {
+			$status = 0
+		    }
+		} elsif ($status < $status{$provides->{$r}}) {
+		    $move = 1;
+		    $status = $status{$provides->{$r}} + 1
+		}
+		$visit{$rpm}{$r} = 1;
+	    }
+	    $status{$rpm} = $status;
+	    $p->[2] = $status
+	}
+    } while ($move);    
+    $run->{todo} = [ sort { $a->[2] <=> $b->[2] } @packages ];
+    @packages
 }