- save status file more often

- dump cache when a failure happen
- fix check needed random rebuild
- use a hash for needed instead of a table
- use the general retry mechanism for srpm deps reinstallation

1 files changed, 163 insertions, 131 deletions

diff --git a/iurt2 b/iurt2
index 0b6c52b..36d51e8 100755
--- a/iurt2
+++ b/iurt2
@@ -303,7 +303,9 @@ if (!$run{debug} && $run{media} || $run{chroot}) {
 my $debug_tag = '_debug' if $run{debug};
 my $chroot_name = "chroot_$run{distro_tag}$debug_tag";
 my $chroot = "$config->{local_home}/$chroot_name";
+$run{chroot_path} = $chroot;
 my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
+$run{chroot_tar} = $chroot_tar;
 if ($run{chroot}) {
     check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot"
 }
@@ -412,7 +414,7 @@ if ($run{shell}) {
     ($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or die "FATAL iurt: could not create temporary chroot";
     add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL iurt: could not add local user";
     $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp";
-    add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi');
+    add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi') or die "FATAL iurt: could not add urpmi and sudo in the chroot";
     add_sudoers(\%run, $luser);
     if ($run{shell}) {
 	print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n";
@@ -484,6 +486,11 @@ do {
 	print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n";
 	# FIXME unfortunately urpmi stalls quite often
 	my $retry = 0;
+	# 
+	# current rpm is sometime segfaulting, and iurt is them blocked and cannot 
+	#
+	# $cache->{failure}{$srpm} = 1;
+	# dump_cache(\%run);
 	retry:
 	if (!$run{chrooted_urpmi}) {
 	    my $match = "urpmi $urpmi_options --root $chroot_tmp";
@@ -496,8 +503,8 @@ do {
 
 	if ($run{chrooted_urpmi}) { 
 	    $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp ";
-	    add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi');
-	    add_media(\%run, $config, $chroot_tmp);
+	    add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi') or next;
+	    add_media(\%run, $config, $chroot_tmp, 'Main', "--distrib $run{chrooted_media}") or next;
 	    if (-d "$local_media/hdlist.cz") {
 		mkdir("$chroot_tmp/iurt_media/");
 		opendir my $dir, $local_media;
@@ -513,7 +520,7 @@ do {
 		    }
 		}
 		next if $next;
-		add_local_media(\%run, $config, $chroot_tmp, "/iurt_media")
+		add_media(\%run, $config, $chroot_tmp, 'iurt_group', "iurt_group file:///iurt_media") or next
 	    }
 	    $run{urpmi_command} = "chroot $chroot_tmp urpmi $urpmi_options "
 	} else {
@@ -555,58 +562,59 @@ do {
 	# 
 	# on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore 
 	# installed packages, retrying several time to be sure something is really broken
-	my $install_deps_retry = 5;
-	while (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm", 
+	if (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm", 
 		\%run, $config,
 		mail => $config->{admin},
 		error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}",  
 		hash => "install_deps_$srpm", 
-		timeout => 900, 
+		timeout => 600, 
 		srpm => $srpm,
 		freq => 1,
 		#cc => $cc, 
-		retry => 10,
+		retry => 2,
 		debug_mail => $run{debug},
 		error_regexp => 'cannot be installed',
-		wait_regexp => 'database locked',
-		wait_callback => $wait_urpmi,
-		log => "$local_spool/log/$srpm/",
-		reopen_log => 1,
-		callback => sub { 
-		    my ($opt, $output) = @_;
-		    print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
+		wait_regexp => { 
+		'database locked' => $wait_urpmi, 
+		'is needed by' => sub { 
+		    print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; 
+		    system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null");
+		    foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } }, 
+	         }, 
+		 wait_callback => $wait_urpmi,
+		 log => "$local_spool/log/$srpm/",
+		 callback => sub { 
+			my ($opt, $output) = @_;
+			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
 # 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error).	    
 # my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g;
-		    my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
-		    # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
-		    @missing_deps or return 1;
-		    while (my $missing_deps = shift @missing_deps) {
-			my $version = shift @missing_deps;
-			my $version2 = shift @missing_deps;
-			$version ||= $version2 || 0;
-			my $p = $pack_provide{$missing_deps} || $missing_deps;
-			my $other_maint = `rpmmon -p "$p"`;
-			chomp $other_maint;
-			print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
-			if ($other_maint && $other_maint ne 'NOT_FOUND') {
-			    $opt->{mail} = "$maintainer, $other_maint";
-			    $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
-			}
-			# remember what is needed, and do not try to recompile until it is available
-			push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
-		    }
-		    0
-		},
-	    )) { 
-	    $install_deps_retry--;
-	    foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
-	    if (!$install_deps_retry || $cache->{needed}{$srpm}) {
+			my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
+			# as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
+			@missing_deps or return 1;
+			while (my $missing_deps = shift @missing_deps) {
+			    my $version = shift @missing_deps;
+			    my $version2 = shift @missing_deps;
+			    $version ||= $version2 || 0;
+			    my $p = $pack_provide{$missing_deps} || $missing_deps;
+			    my $other_maint = `rpmmon -p "$p"`;
+			    chomp $other_maint;
+			    print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
+			    if ($other_maint && $other_maint ne 'NOT_FOUND') {
+				$opt->{mail} = "$maintainer, $other_maint";
+				$opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
+			    }
+			    # remember what is needed, and do not try to recompile until it is available
+			    $cache->{needed}{$srpm}{$missing_deps} = { version => $version, maint => $other_maint || $maintainer };
+			} 
+			0
+		    },
+		)) {
 		$run{status}{$srpm} = 'install_deps_failure';
-		next
+		next;
 	    }
-	}
 	# try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory
 	# running rpm -qa several time seems to fix the problem
+	system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null");
 	foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
 	perform_command("sudo chroot $chroot_tmp rpm -qa", 
 	    \%run, $config,
@@ -623,7 +631,7 @@ do {
 	}
 	if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, 
 		\%run, $config,
-		mail => $maintainer, 
+		#	mail => $maintainer, 
 		error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", 
 		hash => "build_$srpm", 
 		timeout => 18000, 
@@ -670,7 +678,10 @@ do {
 		    goto retry
 		}
 		$cache->{failure}{$srpm} = 1;
+		# 20060615
 		$run{status}{$srpm} = 'build_failure';
+		dump_cache(\%run);
+		dump_status($local_spool, \%run);
 		next
 	    }
 	}
@@ -678,21 +689,22 @@ do {
 	delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm};
 	if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", 
 		\%run, $config,
-		mail => $maintainer, 
+		#	mail => $maintainer, 
 		error => "[REBUILD] binaries packages generated from $srpm do not install correctly", 
 		hash => "binary_test_$srpm", 
 		srpm => $srpm,
 		timeout => 600, 
 		debug_mail => $run{debug},
 		freq => 1, 
-		wait_regexp => 'database locked',
-		wait_callback => $wait_urpmi,
+		wait_regexp => { 'database locked' => $wait_urpmi },
 		error_regexp => 'unable to access',
 		log => "$local_spool/log/$srpm/")) {
 	    $cache->{failure}{$srpm} = 1;
 	    $run{status}{$srpm} = 'binary_test_failure';
 	    next
 	}
+	$run{status}{$srpm} = 'ok';
+	delete $cache->{failure}{$srpm};
 	if ($run{debug}) {
 	    print {$run{LOG}} "iurt: debug mode, skip other packages\n";
 	    exit
@@ -703,7 +715,6 @@ do {
 	    system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_media ($!)\n";
 	} else {
 	    print {$run{LOG}} "iurt: build successful, copying packages to $local_spool.\n";
-	    $run{status}{$srpm} = 'ok';
 	    system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_spool ($!)\n";
 	    if ($run{copy_srpm}) {
 		system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_spool ($!)\n";
@@ -731,18 +742,7 @@ if (open my $file, ">$local_spool/log/wrong_srpm_names.log") {
     close $file
 }
 
-if (open my $file, ">$local_spool/log/status.$run{media}.log") {
-    foreach my $srpm (keys %{$run{status}}) {
-	print $file "$srpm: ";
-	if ($run{status}{$srpm}) {
-	    print $file $run{status}{$srpm}
-	} else {
-	    print $file "unknown"
-	}
-	print $file "\n"
-    }
-    close $file
-}
+dump_status($local_spool, \%run);
 
 if ($config->{rsync_to} && !$run{no_rsync}) { 
     	# remove some old and very big log files not to saturate the server
@@ -846,11 +846,13 @@ sub clean {
     my ($var, $cmd, $kill_cmd, $verbose) = @_;
     my $ps;
     my $i;
+    use POSIX ":sys_wait_h";
     while ($ps = `$cmd "$var"`) {
 	system(qq{$kill_cmd "$var" &>/dev/null});
 	sleep 1;
 	$ps =~ s/\n/,/g;
 	print {$run{LOG}} "Trying to remove previous blocked processes for $var ($ps)\n" if $verbose > 1;
+	waitpid(-1, WNOHANG);
 	return 0 if $i++ > 10
     }
     1
@@ -859,7 +861,7 @@ sub clean {
 sub clean_unionfs {
     my ($unionfs_dir, $run, $r, $union_id) = @_;
     -d "$unionfs_dir/unionfs.$r.$union_id" or return $union_id;
-    print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run->{verbose} > 1;
+    print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$r.$union_id\n" if $run->{verbose} > 1;
     my $nok = 1;
     my $proc = "$unionfs_dir/unionfs.$r.$union_id/proc";
     while ($nok) {
@@ -926,25 +928,25 @@ sub check_mounted {
 sub check_needed {
     my ($srpm) = @_;
     if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 } 
-    my @n;
     my $ok = 1;
-    foreach my $t (@{$cache->{needed}{$srpm}}) {
-	my ($name, $version, $maint) = @$t;
+    foreach my $name (keys %{$cache->{needed}{$srpm}}) {
+	my ($version, $maint) = $cache->{needed}{$srpm}{'version','maint'};
 	my ($p_version) = $provides{$name};
 	if ($p_version) {
 	    next if $version == $p_version;
 	    next if URPM::ranges_overlap($version, $p_version)
 	}
 	$ok = 0;
-	push @n, [ $name, $version ];
+	if ($version) {
+	    $cache->{needed}{$srpm}{$name}{version} = $version;
+	}
 	my $v ||= $version;
 	print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n";
 	# try to recompile it once in a while
-	last if ! $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72;
+	last if $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72;
 	return 1
     }
     delete $cache->{needed}{$srpm};
-    $cache->{needed}{$srpm} = \@n if @n;
     $ok
 }
 
@@ -1090,18 +1092,22 @@ sub perform_command {
     $opt{freq} ||= 24;
     print {$run{LOG}} "Timeout $opt{timeout}\n" if $run{verbose} > 2;
     # from alarm perldoc
-    my ($output, $comment);
+    my ($output, $fulloutput, $comment);
     my ($kill, $pipe);
     if ($opt{debug}) {
 	print "Would have rum $command with a timeout of $opt{timeout}\n";
 	return 1
     }
     local $SIG{PIPE} = sub { print "Broken pipe!\n"; $pipe = 1 };
-    my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log";
     my $retry = $opt{retry} || 1;
     my $call_ret = 1;
-    my ($err, $pid);
+    my ($err, $pid, $try);
+    my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log";
     while ($retry) {
+	$try++;
+	if ($opt{retry} > 1) {
+	    $logfile = "$opt{log}/$opt{hash}-$try.$run->{run}.log";
+	}
 	if ($opt{log}) {
 	    my $parent_pid = $$;
 	    $pid = fork;
@@ -1140,11 +1146,7 @@ sub perform_command {
 	    print {$run{LOG}} "$command\n" if $run{verbose} > 2;
 	    if ($opt{log}) {
 		#$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
-		if ($opt{reopen_log}) {
-		    system("$command 2>&1 >> $logfile");
-		} else {
-		    system("$command >& $logfile");
-		}
+		system("$command 2>&1 >> $logfile");
 	    } else {
 		$output = `$command 2>&1`;
 	    }
@@ -1153,9 +1155,7 @@ sub perform_command {
 	$err = $?;
 	# kill pid watching log file size
 	if ($pid) {
-	    use POSIX ":sys_wait_h";
-	    kill 14, $pid;
-	    waitpid(-1, WNOHANG)
+	    kill_for_good($pid);
 	}
 	if ($@) {
 	    # timed out
@@ -1166,6 +1166,7 @@ sub perform_command {
 	    local $/;
 	    $output = <$log>
 	}
+	$fulloutput .= $output;
 	if (ref $opt{callback}) {
 	    $call_ret = $opt{callback}(\%opt, $output);
 	    $call_ret == -1 and return 1;
@@ -1182,27 +1183,31 @@ sub perform_command {
 	    $comment = "Command failed: $command\n"
 	}
 	# Maybe this has to be put before all the commands altering the $output var
-	if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/m) {
-	    $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
-	    print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n";
-	    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$opt{wait_regexp}\n\n$comment\n$output", 0, 0, $opt{debug_mail});
-	    if ($opt{die}) {
-		dump_cache($run);
-		die "FATAL iurt: $opt{error}."
+	my $inc;
+	if ($opt{wait_regexp}) {
+	    foreach my $wr (keys %{$opt{wait_regexp}}) {
+		if ($output =~ /$wr/m) {
+		    $inc = $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
+		    print {$run->{LOG}} "ERROR iurt: $wr !\n";
+		    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$wr\n\n$comment\n$output", 0, 0, $opt{debug_mail});
+		}
 	    }
-	    $retry--;
-	    return if !$retry
-	} else {
+	}
+	if ($inc) {
+	    $retry += $inc	    
+	} elsif ($call_ret && !$kill && !$err && !$opt{error_regexp} || $fulloutput !~ /$opt{error_regexp}/) {
 	    $retry = 0
+	} else {
+	    $retry-- 
 	}
     }
-    if (!$call_ret || $kill || $err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) {
+    if (!$call_ret || $kill || $err || $opt{error_regexp} && $fulloutput =~ /$opt{error_regexp}/) {
 	if ($opt{log} && $config->{log_url}) {
 	    $comment = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$comment|
 	}
 	my $out;
-	if (length $output < 10000) {
-	    $out = $output
+	if (length $fulloutput < 10000) {
+	    $out = $fulloutput
 	} else { 
 	    $out = "Message too big, see http link for details\n"
 	}
@@ -1216,7 +1221,7 @@ sub perform_command {
 	    }
 	}
 	$cache->{warning}{$opt{hash}}{$opt{mail}}++;
-	print {$run->{LOG}} "\n$comment\n$output\n";
+	print {$run->{LOG}} "\n$comment\n$fulloutput\n";
 	if ($opt{die}) {
 	    dump_cache($run);
 	    die "FATAL iurt: $opt{error}."
@@ -1228,19 +1233,25 @@ sub perform_command {
 
 sub kill_for_good {
     my ($pid) = @_;
+    use POSIX ":sys_wait_h";
     kill 14, $pid;
-    sleep 2;
+    sleep 1;
+    waitpid(-1, WNOHANG);
     if (getpgrp $pid != -1) {
 	kill 15, $pid;
-	sleep 2;
+	sleep 1;
+	waitpid(-1, WNOHANG);
 	if (getpgrp $pid  != -1) {
-	    kill 9, $pid
+	    print STDERR "WARNING iurt: have to kill -9 pid $pid\n";
+	    kill 9, $pid;
+	    sleep 1;
+	    waitpid(-1, WNOHANG);
 	}
     }
 }
 
 sub check_chroot {
-    my ($chroot, $chroot_tar, $run) = @_;
+    my ($chroot, $chroot_tar, $run, $opt) = @_;
     print {$run{LOG}} "iurt: checking basesystem tar\n" if $run{verbose};
     system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot" &> /dev/null});
     if (!clean_chroot($chroot, $run, 1)) {
@@ -1294,10 +1305,6 @@ sub check_pid {
 	    open my $lock2, ">$lockfile.2" or die "FATAL iurt: could not open lock file $lockfile.2";
 	    print $lock2 $$;
 	    close $lock2;
-	} else {
-	    # protection mecchanism to remove dead lock files
-	    unlink "$lockfile.2";
-	    die "FATAL iurt: could not lock pid file (status $status $!)\n";
 	}
     }
     if (!$run->{concurrent_run}) {
@@ -1480,7 +1487,7 @@ sub add_local_user {
     # if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) {
     if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) || system(qq|sudo chroot $chroot_tmp id $luser|)) {
 	print {$run->{LOG}} "ERROR: setting userid $uid to $luser in $chroot_tmp failed, trying to check the chroot\n";
-	check_chroot($chroot, $chroot_tar, $run);
+	check_chroot($run->{chroot_path}, $run->{chroot_tar}, $run);
 	return
     }
     if (system(qq|sudo chroot $chroot_tmp cp -R /home/builder /home/$luser|)) {
@@ -1551,22 +1558,35 @@ sub create_temp_chroot {
 sub add_packages {
     my ($run, $config, $chroot, $user, @packages) = @_;
     my $f;
-    perform_command("sudo $run{urpmi_command} @packages", 
+    if (!perform_command("sudo $run{urpmi_command} @packages", 
 		$run, $config,
 		mail => $config->{admin},
-		timeout => 600, 
+		timeout => 120, 
 		freq => 1,
-		retry => 10,
+		retry => 2,
 		debug_mail => $run->{debug},
 		error_regexp => 'cannot be installed',
-		wait_regexp => 'database locked',
-		wait_callback => sub { 
-		    print {$run->{LOG}} "WARNING iurt: urpmi database locked, waiting...\n";
-		    sleep 30; 
-		    $wait_limit++; 
-		    if ($wait_limit > 10) { 
-			$wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) 
-		    } },)
+		wait_regexp => {
+		    'is needed by' => sub { 
+		        print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; 
+		        system("sudo chroot $chroot rm -rf /var/lib/rpm/__db* &> /dev/null");
+		        foreach (1 .. 3) { system("sudo chroot $chroot rpm -qa &> /dev/null") } 
+		        1
+		        }, 
+		    'database locked' => sub { 
+		        print {$run->{LOG}} "WARNING iurt: urpmi database locked, waiting...\n";
+		        sleep 30; 
+		        $wait_limit++; 
+		        if ($wait_limit > 10) { 
+		    	    $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null});
+			    return
+			} 
+			1
+		  } },)) {
+	print {$run->{LOG}} "ERROR iurt: could not install @packages inside $chroot\n";
+	return 0
+    }
+    1
 }
 
 sub add_sudoers {
@@ -1604,37 +1624,26 @@ sub remove_chroot {
 }
 
 sub add_media {
-    my ($run, $config, $chroot) = @_;
+    my ($run, $config, $chroot, $regexp, $media) = @_;
     print {$run->{LOG}} "iurt: adding media $run{chrooted_media} in chroot $chroot\n";
-    if (!perform_command("sudo chroot $chroot urpmi.addmedia --distrib $run{chrooted_media}", 
-		$run, $config,
-		mail => $config->{admin},
-		timeout => 120, 
-		freq => 1,
-		retry => 5,
-		debug_mail => $run->{debug})) {
-	}
-    check_media_added($chroot, 'Main') or die "FATAL iurt: could not add media into the chroot"
-}
-
-sub add_local_media {
-    my ($run, $config, $chroot, $local_media) = @_;
-    print {$run->{LOG}} "iurt: adding media $local_media in chroot $chroot\n";
-    if (perform_command("sudo chroot $chroot urpmi.addmedia iurt_group file://$local_media", 
+    if (!perform_command("sudo chroot $chroot urpmi.addmedia $media", 
 		$run, $config,
 		mail => $config->{admin},
 		timeout => 120, 
 		freq => 1,
-		retry => 5,
+		retry => 2,
 		debug_mail => $run->{debug})) {
 	}
-    check_media_added($chroot, 'iurt_group') or die "FATAL iurt: could not add local media into the chroot"
+    if (!check_media_added($chroot, $regexp)) { 
+	print "ERROR iurt could not add media into the chroot\n";
+	return
+    } 
+    1
 }
 
 sub check_media_added {
     my ($chroot, $media) = @_;
     my $medias = `sudo chroot $chroot urpmi.removemedia 2>&1`;
-    print "MEDIA $medias\n";
     $medias =~ /one of.* $media/m
 }
 
@@ -1669,10 +1678,16 @@ sub recreate_srpm {
 	debug_mail => $run->{debug},
 	hash => "install_$srpm",
 	retry => $retry,
+	wait_regexp => { 
+	    'cannot write to %sourcedir /usr/src/rpm/SOURCES' => sub {
+		print {$run->{LOG}} "ERROR iurt: chroot seems corrupted, try to rebuild it\n";
+		check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run)
+	    }		
+	},
 	callback => sub { 
 	    my ($opt, $output) = @_;
 	    print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug};
-	    if ($output =~ /warning: group .* does not exist - using root/i) {
+	    if ($output =~ /warning: (group|user) .* does not exist - using root/i) {
 		return 1
 	    } elsif ($output =~ /user $luser does not exist/) {
 		print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1;
@@ -1686,7 +1701,7 @@ sub recreate_srpm {
 	    \%run, $config, %opt)) {
 	print {$run->{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run->{debug};
 	if ($opt{retry}) {
-	    check_chroot($chroot, $chroot_tar, \%run) or return;
+	    check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) or return;
 	    return -1
 	}
 	return
@@ -1767,3 +1782,20 @@ sub order_packages {
     $run->{todo} = [ sort { $a->[2] <=> $b->[2] } @packages ];
     @packages
 }
+
+sub dump_status {
+    my ($local_spool, $run) = @_;
+    if (open my $file, ">$local_spool/log/status.$run->{media}.log") {
+	foreach my $srpm (sort keys %{$run->{status}}) {
+	    print $file "$srpm: ";
+	    if ($run{status}->{$srpm}) {
+		print $file $run->{status}{$srpm}
+	    } else {
+		print $file "unknown"
+	    }
+	    print $file "\n"
+	}
+	close $file
+    }
+}
+