create a subdir per user for chroot, add --clean option

1 files changed, 239 insertions, 150 deletions

diff --git a/iurt2 b/iurt2
index 4ab0d39..ad198a8 100755
--- a/iurt2
+++ b/iurt2
@@ -46,7 +46,7 @@ $run{todo} = [ ];
 @params = ( 
     #    [ "one letter option", "long name option", "number of args (-X means ´at least X´)", "help text", "function to call", "log info"]
     [ "", "$program_name", 0, "[--cache] [--concurrent-run] [--config foo value] [--warn] [--verbose integer]
-            [--copy_srpm] [--debug] [--distro] [--no_rsync] [--shell] [--stop {p|c|i|l|b|a|s}]
+            [--copy_srpm] [--debug] [--distro] [--no_rsync] [--clean user1 user2 user3] [--clean_all] [--shell] [--stop {p|c|i|l|b|a|s}]
  	    [--use-system-distrib] [--dir] [--help foo?] [--log filename]  [--unionfs]
  	    [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename]  [--unionfs]
  	    {--config_help |
@@ -69,6 +69,12 @@ $run{todo} = [ ];
     [ "c", "chroot", 0, "", 
     "Check chroot and update it if needed",
     sub { $run{chroot} = 1 }, "Activating chroot updating" ],
+    [ "", "clean-all", 0, "", 
+    "Clean all remaining chroots for all the users",
+    sub { $run{clean_all} = 1 }, "Activating clean chroot flag" ],
+    [ "", "clean", -1, "<user 1> <user 2> ... <user n>", 
+    "Clean remaining chroot before runing",
+    sub { $run{clean} = \@_ }, "Activating clean chroot flag" ],
     [ "", "concurrent-run", 0, "", 
     "Allow several iurt to run on different machines (slower)",
     sub { $run{concurrent_run} = 1 }, "Activating concurrent run checks" ],
@@ -257,14 +263,15 @@ if ($run{distro} ne 'cooker') {
 # cache file name is needed early to remove the manual lock file if the lock mechanism does not work
 my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$run{media}.cache";
 $run{cachefile} = $cachefile;
-if (!$run{debug} && $run{media}) {
+if (!$run{debug} && $run{media} || $run{chroot}) {
     $run{pidfile_home} = "$config->{cache_home}/";
     $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$run{media}";
     check_pid(\%run)
 }
 
 my $debug_tag = '_debug' if $run{debug};
-my $chroot = "$config->{local_home}/chroot_$run{distro_tag}$debug_tag";
+my $chroot_name = "chroot_$run{distro_tag}$debug_tag";
+my $chroot = "$config->{local_home}/$chroot_name";
 my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
 if ($run{chroot}) {
     check_chroot($chroot, $chroot_tar, \%run)
@@ -303,15 +310,8 @@ $to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{ext
 
 dump_cache(\%run);
 
-if (!@{$run{todo}} && !$run{debug} && !$run{shell}) {
-    print {$run{LOG}} "iurt: nothing to do\n";
-    unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
-    exit
-}
 print {$run{LOG}} "iurt: will try to compile $to_compile packages\n" if $run{verbose} > 1;
 
-exit if !$run{rebuild};
-
 my ($fulldate, $daydate) = get_date();
 if ($run{use_cache}) {
     $run{run} = $cache->{run};
@@ -320,6 +320,71 @@ if ($run{use_cache}) {
     $run{run} = "0.$fulldate"
 }
 print {$run{LOG}} "iurt: using $run{run} as chroot extension\n" if $run{verbose} > 4;
+$run{user} = $ENV{SUDO_USER} || $ENV{USER};
+$run{uid} = getpwnam $run{user};
+print {$run{LOG}} "iurt: using local user $run{user}, id $run{uid}\n" if $run{verbose} > 3;
+my $luser = $run{user} || 'builder';
+
+my $unionfs_dir;
+if ($run{unionfs}) {
+    # FIXME need to grep /proc/modules not ot try to load it if already loaded
+    open my $modules, '/proc/modules';
+    my $ok;
+    while (my $m = <$modules>) {
+	if ($m =~ /unionfs/) {
+	    $ok = 1;
+	    last
+	}
+    }
+    if (!$ok) {
+	print {$run{LOG}} "iurt: adding unionfs module\n" if $run{verbose} > 0;
+	system("sudo /sbin/depmod -a");
+	system("sudo /sbin/modprobe -f unionfs");
+    }
+    $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag/";
+    remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs);
+    $unionfs_dir = "$unionfs_dir/$run{user}/";
+    -d $unionfs_dir or mkdir $unionfs_dir
+}
+
+my (%done, $wait_limit, $done);
+$run{done} = \%done;
+my $home = $config->{local_home};
+my $union_id = 1;
+my $unionfs_tmp = $run{unionfs};	
+my $chroot_tmp = "$config->{local_home}/chroot_tmp";
+if (!-d $chroot_tmp) {
+    mkdir $chroot_tmp
+} else {
+    remove_chroot(\%run, $chroot_tmp, \&clean_all_chroot_tmp, $chroot_name);
+}
+$chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}";
+if (!-d $chroot_tmp) {
+    mkdir $chroot_tmp
+}
+$chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}/$chroot_name.$run{run}";
+
+# now exit if there is nothing to do and it was just a cleaning pass
+if (!@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild}) {
+    print {$run{LOG}} "iurt: nothing to do\n";
+    unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
+    exit
+}
+
+my $df = df $home;
+if ($df->{per} == 100) {
+    die "FATAL iurt: not enough space on the filesystem, only $df->{bavail}KB on $home, full at $df->{per}%"
+}
+
+if ($run{shell}) {
+    my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id);
+    add_local_user($chroot_tmp, \%run, $luser, $run{uid});
+    add_sudo(\%run, $config, $chroot_tmp, $luser);
+    if ($run{shell}) {
+	print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n";
+	exec "sudo chroot $chroot_tmp /bin/su $luser -c bash"
+    }
+}
 
 $config->{local_upload} ||= $config->{local_home};
 my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}";
@@ -347,26 +412,6 @@ if (!dump_rpmmacros("$chroot/home/builder/.rpmmacros")) {
      check_chroot($chroot, $chroot_tar, \%run);
      dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros"
 }
-my $unionfs_dir;
-if ($run{unionfs}) {
-    # FIXME need to grep /proc/modules not ot try to load it if already loaded
-    open my $modules, '/proc/modules';
-    my $ok;
-    while (my $m = <$modules>) {
-	if ($m =~ /unionfs/) {
-	    $ok = 1;
-	    last
-	}
-    }
-    if (!$ok) {
-	print {$run{LOG}} "iurt: adding unionfs module\n" if $run{verbose} > 0;
-	system("sudo /sbin/depmod -a");
-	system("sudo /sbin/modprobe -f unionfs");
-    }
-    $unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag";
-    clean_all_unionfs($unionfs_dir);
-    -d $unionfs_dir or mkdir $unionfs_dir
-}
 
 my $s = sub { 
     if ($run{main}) {
@@ -383,35 +428,6 @@ $SIG{TERM} = $s;
 $SIG{INT} = $s;
 $run{main} = 1;
 
-my %done;
-$run{done} = \%done;
-my $wait_limit;
-my $done;
-my $home = $config->{local_home};
-my $union_id = 1;
-my $unionfs_tmp = $run{unionfs};	
-my $chroot_tmp = "$chroot.$run{run}";
-
-my $df = df $home;
-if ($df->{per} == 100) {
-    die "FATAL iurt: not enough space on the filesystem, only $df->{bavail}KB on $home, full at $df->{per}%"
-}
-
-$run{user} = $ENV{SUDO_USER} || $ENV{USER};
-$run{uid} = getpwnam $run{user};
-print {$run{LOG}} "iurt: using local user $run{user}, id $run{uid}\n" if $run{verbose} > 3;
-my $luser = $run{user} || 'builder';
-
-if ($run{shell}) {
-    my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id);
-    add_local_user(\%run, $luser, $run{uid});
-    add_sudo($chroot_tmp, $luser);
-    if ($run{shell}) {
-	print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n";
-	exec "sudo chroot $chroot_tmp /bin/su $luser -c bash"
-    }
-}
-
 foreach (my $i ; $i < @{$run{todo}}; $i++) {
 	my ($dir, $srpm) = @{$run{todo}[$i]};
 	$done{$srpm} and next;
@@ -444,10 +460,11 @@ retry:
 	}
 	#($maintainer, $cc) = ($config->{admin},'');
 
-	add_local_user(\%run, $luser, $run{uid});
+	print {$run{LOG}} "iurt: adding local user to the chroot...\n" if $run{verbose};
+	add_local_user($chroot_tmp, \%run, $luser, $run{uid});
 
 	# recreate a new srpm for buildarch condition in the spec file
-	print {$run{LOG}} "Copying $srpm to $chroot_tmp\n" if $run{verbose} > 1;
+	print {$run{LOG}} "iurt: copying $srpm to $chroot_tmp\n" if $run{verbose} > 1;
 	perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/", 
 	        \%run, $config,
 		mail => $config->{admin}, 
@@ -472,6 +489,7 @@ retry:
 			1
 		    }
 		);
+	print {$run{LOG}} "iurt: recreating src.rpm...\n" if $run{verbose};
 	if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"}, 
 		\%run, $config, %opt)) {
 	        print {$run{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run{debug};
@@ -489,7 +507,7 @@ retry:
 		error => "[REBUILD] cannot create $srpm in $chroot_tmp", 
 		debug_mail => $run{debug},
 		hash => "create_$srpm") or next;
-	print {$run{LOG}} "Installing build dependencies of $srpm...\n" if $run{verbose} > 1;
+	print {$run{LOG}} "iurt: installing build dependencies of $srpm...\n" if $run{verbose};
 	
 	if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm", 
 		\%run, $config,
@@ -500,16 +518,24 @@ retry:
 		srpm => $srpm,
 		freq => 1,
 		#cc => $cc, 
+		retry => 10,
 		debug_mail => $run{debug},
 		error_regexp => 'cannot be installed',
 		wait_regexp => 'database locked',
-		wait_callback => sub { $wait_limit++; if ($wait_limit > 10) { $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) } },
+		wait_callback => sub { 
+		    print {$run{LOG}} "WARNING iurt: urpmi database locked, waiting...\n" if $run{debug};
+		    sleep 30; 
+		    $wait_limit++; 
+		    if ($wait_limit > 10) { 
+			$wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) 
+		    } },
 		log => "$local_spool/log/$srpm/",
 		callback => sub { 
 			my ($opt, $output) = @_;
 			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
 			my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
-			@missing_deps or return;
+			# as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
+			@missing_deps or return 1;
 			while (my $missing_deps = shift @missing_deps) {
 			    my $version = shift @missing_deps;
 			    my $version2 = shift @missing_deps;
@@ -525,6 +551,7 @@ retry:
 			    # remember what is needed, and do not try to recompile until it is available
 			    push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
 			}
+			1
 		},
 	)) { 
 	    $run{status}{$srpm} = 'install_deps_failure';
@@ -544,7 +571,7 @@ retry:
 	print {$run{LOG}} "Compiling $srpm\n" if $run{verbose}; 
 	my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm";
 	if ($run{stop}) {
-	    add_sudo($chroot_tmp, $luser);
+	    add_sudo(\%run, $config, $chroot_tmp, $luser);
 	    $command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec"
 	}
 	if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, 
@@ -574,6 +601,7 @@ retry:
 				$cache->{no_unionfs}{$srpm} = 1;
 				return 1
 			    }
+			    1
 			}, 
 			freq => 1) && !glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") {
 		    # FIXME
@@ -667,7 +695,7 @@ if ($config->{rsync_to} && !$run{no_rsync}) {
 # one last try to clean
 print {$run{LOG}} "iurt: try to clean remaining unionfs\n" if $run{verbose};
 if ($run{unionfs}) {
-    clean_all_unionfs($unionfs_dir)
+    remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs)
 }
 unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
 exit;
@@ -699,6 +727,17 @@ sub clean_all_unionfs {
     closedir $dir
 }
 
+sub clean_all_chroot_tmp {
+    my ($chroot_dir, $prefix) = @_;
+    print {$run{LOG}} "Cleaning old chroot remaining dir in $chroot_dir\n" if $run{verbose};
+    opendir my $dir, $chroot_dir or die "FATAL iurt: could not open $chroot_dir ($!)";
+    foreach (readdir $dir) {
+	/$prefix.*$/ or next;
+	clean_chroot_tmp($chroot_dir, $_);
+    }
+    closedir $dir
+}
+
 sub clean_chroot {
     my ($chroot, $run, $only_clean) = @_;
     if (-d $chroot) {
@@ -755,7 +794,7 @@ sub clean_unionfs {
     if (system("sudo umount $unionfs_dir/unionfs.$run.$union_id/proc &>/dev/null") && $run{verbose} > 1) { 
 	print {$run{LOG}} "ERROR iurt: could not umount /proc in $unionfs_dir/unionfs.$run.$union_id\n" 
     }
-    print "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run{verbose} > 1;
+    print {$run{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run{verbose} > 1;
     my $nok = 1;
     while ($nok) {
 	foreach my $t ("unionfs",'tmpfs') {
@@ -781,6 +820,18 @@ sub clean_unionfs {
     $union_id
 }
 
+sub clean_chroot_tmp {
+    my ($chroot_dir, $dir) = @_;
+    my $d = "$chroot_dir/$dir";
+    if (system("sudo umount $d/proc &>/dev/null") && $run{verbose} > 1) { 
+	print {$run{LOG}} "ERROR iurt: could not umount /proc in $d/\n" 
+    }
+    print {$run{LOG}} "iurt: cleaning $d\n" if $run{verbose};
+    system("sudo /sbin/fuser -k $d &> /dev/null");
+    print {$run{LOG}} "iurt: removing $d\n" if $run{verbose};
+    system(qq{sudo rm -rf $d});
+}
+
 sub check_mounted {
     my ($mount_point, $type) = @_;
     open my $mount, '/proc/mounts' or die 'FATAL iurt: could not open /proc/mounts';
@@ -965,90 +1016,96 @@ sub perform_command {
     }
     local $SIG{PIPE} = sub { print "Broken pipe!\n"; $pipe = 1 };
     my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log";
-    my $pid;
-    if ($opt{log}) {
-	my $parent_pid = $$;
-	$pid = fork;
-	my $tot_time;
-	if (!$pid) {
-	    print {$run{LOG}} "Forking to monitor log size\n" if $run{verbose} > 2;
-	    $run->{main} = 0;
-	    local $SIG{ALRM} = sub { exit };
-	    $tot_time += sleep 30;
-	    my $size_limit = $config->{log_size_limit};
-	    $size_limit =~ s/k/000/i;
-	    $size_limit =~ s/M/000000/i;
-	    $size_limit =~ s/G/000000000/i;
-	    while ($tot_time < $opt{timeout}) {
-		my (@stat) = stat $logfile;
-		if ($stat[7] > $size_limit) {
-		    print {$run{LOG}} "WARNING: killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})\n";
-		    kill 14, "-$parent_pid";
-		    exit
-		}
-		my $df = df $opt{log} ;
-		if ($df->{per} == 100) {
-		    print {$run{LOG}} "WARNING: killing current command because running out of disk space (only $df->{bavail}KB left)\n";
-		    kill 14, "-$parent_pid";
-		    exit
-		}
+    my $retry = $opt{retry} || 1;
+    my $call_ret = 1;
+    my ($err, $pid);
+    while ($retry) {
+	if ($opt{log}) {
+	    my $parent_pid = $$;
+	    $pid = fork;
+	    my $tot_time;
+	    if (!$pid) {
+		print {$run{LOG}} "Forking to monitor log size\n" if $run{verbose} > 2;
+		$run->{main} = 0;
+		local $SIG{ALRM} = sub { exit };
 		$tot_time += sleep 30;
+		my $size_limit = $config->{log_size_limit};
+		$size_limit =~ s/k/000/i;
+		$size_limit =~ s/M/000000/i;
+		$size_limit =~ s/G/000000000/i;
+		while ($tot_time < $opt{timeout}) {
+		    my (@stat) = stat $logfile;
+		    if ($stat[7] > $size_limit) {
+			print {$run{LOG}} "WARNING: killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})\n";
+			kill 14, "-$parent_pid";
+			exit
+		    }
+		    my $df = df $opt{log} ;
+		    if ($df->{per} == 100) {
+			print {$run{LOG}} "WARNING: killing current command because running out of disk space (only $df->{bavail}KB left)\n";
+			kill 14, "-$parent_pid";
+			exit
+		    }
+		    $tot_time += sleep 30;
+		}
+		exit
 	    }
-	    exit
 	}
-    }
-    eval {
-	local $SIG{ALRM} = sub { print "Timeout!\n"; $kill = 1; die "alarm\n" }; # NB: \n required
-	alarm $opt{timeout};
-	print {$run{LOG}} "$command\n" if $run{verbose} > 2;
-	if ($opt{log}) {
-	    #$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
-	    system("$command &> $logfile");
+	eval {
+	    local $SIG{ALRM} = sub { print "Timeout!\n"; $kill = 1; die "alarm\n" }; # NB: \n required
+	    alarm $opt{timeout};
+	    print {$run{LOG}} "$command\n" if $run{verbose} > 2;
+	    if ($opt{log}) {
+		#$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
+		system("$command &> $logfile");
+	    } else {
+		$output = `$command 2>&1`;
+	    }
+	    alarm 0
+	};
+	$err = $?;
+	# kill pid watching log file size
+	if ($pid) {
+	    use POSIX ":sys_wait_h";
+	    kill 14, $pid;
+	    waitpid(-1, WNOHANG)
+	}
+	if ($@) {
+	    # timed out
+	    die "FATAL iurt: unexpected signal" unless $@ eq "alarm\n";   # propagate unexpected errors
+	}
+	# Keep the run first on the harddrive so that one can check the command status tailing it
+	if ($opt{log} && open my $log, $logfile) {
+	    local $/;
+	    $output = <$log>
+	}
+	if (ref $opt{callback}) {
+	    $call_ret = $opt{callback}(\%opt, $output);
+	    $call_ret == -1 and return 1
+	}
+	if ($kill) {
+	    $output = "Command has been killed after $opt{timeout} seconds: $command\n$output";
+	    my ($cmd_to_kill) = $command =~ /sudo(?: chroot \S+)? (.*)/;
+	    clean_process($cmd_to_kill, $run{verbose})
+	} elsif ($pipe) {
+	    $output = "Command receives a broken pipe: $command\n$output";
+	    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: broken pipe", "$output", 0, 0, $opt{debug_mail});
 	} else {
-	    $output = `$command 2>&1`;
+	    $output = "Command failed: $command\n$output"
 	}
-	alarm 0
-    };
-    my $err = $?;
-    # kill pid watching log file size
-    if ($pid) {
-	use POSIX ":sys_wait_h";
-	kill 14, $pid;
-	waitpid(-1, WNOHANG)
-    }
-    if ($@) {
-	# timed out
-	die "FATAL iurt: unexpected signal" unless $@ eq "alarm\n";   # propagate unexpected errors
-    }
-    # Keep the run first on the harddrive so that one can check the command status tailing it
-    if ($opt{log} && open my $log, $logfile) {
-	local $/;
-	$output = <$log>
-    }
-    my $call_ret = 1;
-    if (ref $opt{callback}) {
-	$call_ret = $opt{callback}(\%opt, $output);
-	$call_ret == -1 and return;
-    }
-    if ($kill) {
-	$output = "Command has been killed after $opt{timeout} seconds: $command\n$output";
-	my ($cmd_to_kill) = $command =~ /sudo(?: chroot \S+)? (.*)/;
-	clean_process($cmd_to_kill, $run{verbose})
-    } elsif ($pipe) {
-        $output = "Command receives a broken pipe: $command\n$output";
-	sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: broken pipe", "$output", 0, 0, $opt{debug_mail});
-    } else {
-	$output = "Command failed: $command\n$output"
-    }
-    if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/) {
-	$opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
-	print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n";
-	sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$opt{wait_regexp}\n\n$output", 0, 0, $opt{debug_mail});
-	if ($opt{die}) {
-	    dump_cache($run);
-	    die "FATAL iurt: $opt{error}."
+	if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/) {
+	    $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
+	    print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n";
+	    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$opt{wait_regexp}\n\n$output", 0, 0, $opt{debug_mail});
+	    if ($opt{die}) {
+		dump_cache($run);
+		die "FATAL iurt: $opt{error}."
+	    }
+	    $retry--;
+	    return if !$retry
+	} else {
+	    $retry = 0
 	}
-	return 0
     }
     if (!$call_ret || $kill || $err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) {
 	if ($opt{log} && $config->{log_url}) {
@@ -1318,11 +1375,11 @@ sub get_date {
 }
 
 sub add_local_user {
-    my ($run, $luser, $uid) = @_;
+    my ($chroot_tmp, $run, $luser, $uid) = @_;
     # change the builder user to the local user id
     # FIXME it seems that unionfs does not handle well the change of the uid of files
     # if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) {
-    if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|)) {
+    if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) && system(qq|sudo chroot $chroot_tmp id $luser|)) {
 	print {$run->{LOG}} "ERROR: setting userid to builder user in chroot failed, trying to check the chroot\n";
 	check_chroot($chroot, $chroot_tar, $run);
     }
@@ -1370,15 +1427,31 @@ sub create_temp_chroot {
 }
 
 sub add_sudo {
-    my ($chroot, $user) = @_;
+    my ($run, $config, $chroot, $user) = @_;
     my $f;
+    perform_command("sudo urpmi $urpmi_options --root $chroot_tmp sudo urpmi", 
+		$run, $config,
+		mail => $config->{admin},
+		timeout => 600, 
+		freq => 1,
+		retry => 10,
+		debug_mail => $run->{debug},
+		error_regexp => 'cannot be installed',
+		wait_regexp => 'database locked',
+		wait_callback => sub { 
+		    print {$run->{LOG}} "WARNING iurt: urpmi database locked, waiting...\n";
+		    sleep 30; 
+		    $wait_limit++; 
+		    if ($wait_limit > 10) { 
+			$wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) 
+		    } },);
     if (system("sudo urpmi $urpmi_options --root $chroot_tmp sudo urpmi")) { 
 	die "FATAL iurt: could not install sudo in the $chroot_tmp ($!)"
     }
     my $file = "$chroot/etc/sudoers";
     my $f;
     if (!open $f, qq{| sudo sh -c "cat > $file"}) {
-	print {$run{LOG}} "ERROR iurt: could not open $file ($!)\n";
+	print {$run->{LOG}} "ERROR iurt: could not open $file ($!)\n";
 	return 0
     }
     print $f qq{Cmnd_Alias RPM=/bin/rpm,/usr/sbin/urpmi,/usr/sbin/urpme,/usr/sbin/urpmi.addmedia
@@ -1386,7 +1459,23 @@ root    ALL=(ALL) ALL
 $user   ALL=(ALL) NOPASSWD:RPM
 };
     close $f;
-    print {$run{LOG}} "iurt: adding sudo for /bin/rpm, /usr/sbin/urpmi and /usr/sbin/urpme\n";
+    print {$run->{LOG}} "iurt: adding sudo for /bin/rpm, /usr/sbin/urpmi and /usr/sbin/urpme\n";
     -f $file or return 0;
     1
 }
+
+sub remove_chroot {
+    my ($run, $dir, $func, $prefix) = @_;
+    if ($run->{clean_all}) {
+	opendir my $chroot_dir, $dir;
+	foreach (readdir $chroot_dir) {
+	    next if !-d || /\.{1,2}/;
+	    $func->("$dir/$_", $prefix)
+	}
+    } else {
+	foreach my $user (@{$run->{clean}}) {
+	    print {$run->{LOG}} "iurt: cleaning old chroot for $user\n";
+	    $func->("$dir/$user", $prefix)
+	}
+    }
+ }