path: root/iurt2

#!/usr/bin/perl
#
# Copyright (C) 2005 Mandrakesoft
# Copyright (C) 2005,2006 Mandriva
# 
# Author: Florent Villard <warly@mandriva.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# compare and rebuild packages on different architecture
#
# TODO
#
# - use a cache (rpmctl cache for example) to find maintainer
# - add icecream compilation support
# - add a --group option to compile a set of packages (in progress)
# - add a function to update a packages when it obviously need to be recompile
# - Maybe call the function from the initial todo list (thus making the argument ordering important)
#
use strict;
use RPM4::Header;
use Data::Dumper;
use URPM;
use File::NCopy qw(copy);
use MIME::Words qw(encode_mimewords);
# I did not manage to make locks work over the network
#use File::lockf;
use Mkcd::Commandline qw(parseCommandLine usage);
use MDK::Common;
use Filesys::Df qw(df);

my $program_name = 'iurt2';
# sessing parameters
my $arg = @ARGV;
my (@params, %run);
my %arch_comp = (
    'i586' => { 'i386' => 1, 'i486' => 1, 'i586' => 1 },
    'i686' => { 'i386' => 1, 'i486' => 1, 'i586' => 1, 'i686' => 1 },
    'x86_64' => { 'x86_64' => 1 },
    'ppc' => { 'ppc' => 1 },
    'ppc64' => { 'ppc' => 1, 'ppc64' => 1 },
);

$run{todo} = [ ];
@params = ( 
    #    [ "one letter option", "long name option", "number of args (-X means ´at least X´)", "help text", "function to call", "log info"]
    #
    # no_rsync, config_help and copy_srpm kept for compatibility reasons
    #
    [ "", "$program_name", 0, "[--cache] [--chrooted-urpmi <media prefix>] [--concurrent-run] [--config foo value] [--warn] [--verbose integer]
            [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}]
 	    [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs]
 	    [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename]  [--unionfs] [--status] [--ignore-failure]
 	    {--config_help |
	    --chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } |
 	    --rebuild {cooker|2006.0|community/2006.0|...} {i586|x86_64|ppc|...} {filename1.src.rpm} {filename2.src.rpm} ... {filenamen.src.rpm} }", 
    "$program_name is a perl script to rebuild automatically several rpm in chroot, given a sourcerpm repository, and mail authors or rebuilder when problems occurs.", 
    sub { $arg or usage($program_name, \@params) }, "" ],
    [ "", "distro", 1, "<distro>", 
    "Set the distribution",
    sub { ($run{distro}) = @_; 1 }, "Setting the distribution" ],
    [ "a", "arch", 1, "<architecture>", 
    "Set the architecture",
    sub { ($run{my_arch}) = @_; 1 }, "Setting architecture" ],
    [ "", "cache", 0, "", 
    "Use the global cache file",
    sub { $run{use_cache} = 1 }, "Activating cache use" ],
    [ "", "copy-srpm", 0, "", 
    "Copy also the regenerated SRPM",
    sub { $run{copy_srpm} = 1 }, "Activating the copy_srpm mode" ],
    [ "", "copy_srpm", 0, "", 
    "Copy also the regenerated SRPM",
    sub { $run{copy_srpm} = 1 }, "Activating the copy_srpm mode" ],
    [ "c", "chroot", 0, "", 
    "Check chroot and update it if needed",
    sub { $run{chroot} = 1 }, "Activating chroot updating" ],
    [ "", "chrooted-urpmi", 1, "<media prefix>", 
    "Create urpmi media inside the chroot instead of using --root (media prefix is like http:///server.mandriva.com/dis/)",
    sub { $run{chrooted_urpmi} = shift }, "Activating chroot updating" ],
    [ "", "clean-all", 0, "", 
    "Clean all remaining chroots for all the users",
    sub { $run{clean_all} = 1 }, "Activating clean chroot flag" ],
    [ "", "clean", -1, "<user 1> <user 2> ... <user n>", 
    "Clean remaining chroot before runing",
    sub { $run{clean} = \@_ }, "Activating clean chroot flag" ],
    [ "", "concurrent-run", 0, "", 
    "Allow several iurt to run on different machines (slower)",
    sub { $run{concurrent_run} = 1 }, "Activating concurrent run checks" ],
    [ "d", "dir", -1, "", 
    "Directory where to find packages to rebuild", 
    sub { $run{extra_dir} = \@_; 1 }, "Adding extra source packages directories" ],
    [ "", "config", 2, "<configuration keyword> <value>", 
    "Override a configuration file variable",
    sub { my ($key, $value) = @_; $run{config}{$key} = $value }, "Overriding configuration variable" ],
    [ "", "config-help", 0, "", 
    "Explain configuration files keywords", 
    sub { $run{config_usage} = 1 }, "Activating debug mode" ],
    [ "", "config_help", 0, "", 
    "Explain configuration files keywords", 
    sub { $run{config_usage} = 1 }, "Activating debug mode" ],
    [ "", "debug", 0, "", 
    "Activate debug mode", 
    sub { $run{debug} = 1 }, "Activating debug mode" ],
    [ "g", "group", 0, "", 
    "Activate group mode, packages will be compiled as a global set, not as individual packages", 
    sub { $run{group} = 1 }, "Activating the group mode" ],
    [ "", "ignore-failure", 0, "", 
    "Do not take into account the failure cache, try to recompile all the packages not synchronized", 
    sub { $run{ignore_failure} = 1 }, "Activating the mode ignoring previous failure" ],
    [ "u", "unionfs", 0, "", 
    "Activate unionfs mode", 
    sub { $run{unionfs} = 1 }, "Activating unionfs mode" ],
    [ "l", "log", 1, "<log file>", 
    "Log file.", 
    sub { 
	$run{log} = pop @_; 
	open $run{LOG}, ">$run{log}" or die "unable to open $run{log}\n";
	print *{$run{LOG}}, "command line: @ARGV\n";
	1
    }, "Log file" ],
    [ "m", "media", -1, "<media 1> <media 2> ... <media 3>", 
    "Media to rebuild", 
    sub { ($run{media}) = @_; 1 }, "Adding a media to rebuild" ],
    [ "n", "no", 0, "", 
    "Perform all the check but do not compile anything", 
    sub { ($run{no_compile}) = 1 }, "Setting the no compilation flag" ],
    [ "r", "rebuild", -2, "<distro> <architecture> <srpm 1> <srpm 2> ... <srpm n>", 
    "Rebuild the packages, e.g. $program_name -r cooker x86_64 /home/foo/rpm/SRPMS/foo-2.3-12mdv2007.0.src.rpm", 
    sub { 
	$run{rebuild} = 1; 
	$run{distro} = shift @_;
       	$run{my_arch} = shift @_;
	foreach (@_) {
	    my ($path, $srpm);
	    if (m,(.*/)([^/]*.src.rpm)$, && -f $_) {
		($path, $srpm) = ( $1, $2 )
	    } elsif (m,([^/]*.src.rpm)$, && -f $_) {
		($path, $srpm) = ( './', $1 )
	    } else {
		die "FATAL iurt: $_ does not seems to be a SRPM\n"
	    }
	    my $hdr = RPM4::Header->new($_);
	    if (check_arch($hdr)) {
		print {$run{LOG}} "iurt: force build for $2 (from $1)\n";
		push @{$run{todo}}, [ $path, $srpm, 1 ]
	    } else {
		print {$run{LOG}} "ERROR iurt: $_ could not be build on $run{my_arch}, ignored.\n"
	    }
	}
	1
	}, "Activating rebuild mode" ],
    [ "", "upload", [
	["", "upload", 0, "[options]", 
	"Upload the rebuild packages", 
	sub {   my ($tmp) = @_;
	    $tmp->[0] ||= {};
	    1
	}, "Setting upload options"],
	[ "m", "markrelease", 0, "", 
	"Mark SVN directory when uploading the packages", 
	sub { $run{markrelease} = 1 }, "Adding markrelease repsys option" ],
	[ "s", "source", 0, "", 
	"Upload the source package as wells", 
	sub { $run{source_upload} = 1 }, "Setting source flag for upload" ],
    ], "[options]", 
    "Upload the rebuild packages", 
    sub { $run{upload} = 1 }, "Setting the upload flag" ],
    [ "", "no_rsync", 0, "", 
    "Do not send build log to the distant rsync server", 
    sub { $run{no_rsync} = 1 }, "Setting the no rsync warn flag" ],
    [ "", "no-rsync", 0, "", 
    "Do not send build log to the distant rsync server", 
    sub { $run{no_rsync} = 1 }, "Setting the no rsync warn flag" ],
    [ "", "use-system-distrib", 1, "<media>", 
    "Use the current system urpmi configuration", 
    sub { $run{use_system_distrib} = shift; 1 }, "Setting system distrib for urpmi configuration" ],
    [ "v", "verbose", 1, "<verbose level>", 
    "Give more info messages about what is going on (level from 1 to 10)", 
    sub { $run{verbose} = $_[0]; 1 }, "Setting verbose level" ],
    [ "w", "warn", 0, "", 
    "Warn maintainer of the packages about problem in the rebuild", 
    sub { $run{warn} = 1 ; 1 }, "Setting warn flag to warn maintainers" ],
    [ "", "shell", 0, "", 
    "Dump to a shell into the newly created chroot with sudo on rpm, urpmi, urpme and urpmi.addmedia", 
    sub { 
	($run{shell}) = 1; 
	1 }, "Setting option to dump to a shell" ],
    [ "", "stop", 1, "<rpm step>", 
    "Perform rpm -b<rpm step> (p c i l b a s) instead of rpm -ba and then open a shell in the chroot", 
    sub { 
	($run{stop}) = @_; 
	1 
    }, "Setting rpm build option" ],
    [ "", "status", 1, "<mail>", 
    "Send a status mail to the provided mail address", 
    sub { 
	($run{status_mail}) = @_; 
	1 
    }, "Setting status mail option" ],
);

open(my $LOG, ">&STDERR");
$run{LOG} = $LOG;

my $todo = parseCommandLine($program_name, \@ARGV, \@params);
@ARGV and usage($program_name, \@params, "@ARGV, too many arguments");
foreach my $t (@$todo)  {
    print {$run{LOG}} "$program_name: $t->[2]\n" if $run{verbose} > 5;
    &{$t->[0]}(@{$t->[1]}) or print {$run{LOG}} "ERROR: $t->[2]\n";
}

$run{distro_tag} = $run{distro};
$run{distro_tag} =~ s,/,-,g;

my $real_arch = `uname -m`;
chomp $real_arch;
my $HOME = $ENV{HOME};
my $configfile = "$HOME/.iurt.$run{distro_tag}.conf";
print {$run{LOG}} "iurt: loading config file $configfile\n" if $run{verbose} > 1;
my $config;
if (-f $configfile) {
    $config = do $configfile or die "FATAL iurt: syntax error in $configfile";
} else {
    $config = {}
}
my $urpmi_options = "-v --no-verify-rpm --nolock --auto --ignoresize";
if ($run{use_system_distrib}) {
    $config->{basesystem_media_root} ||= $run{use_system_distrib}
} elsif ($run{chrooted_urpmi}) {
    my ($host) = $run{chrooted_urpmi} =~ m,(?:file|http|ftp)://([^/]*),;
    my ($name,$aliases,$addrtype,$length,@addrs) = gethostbyname $host;
    my $ip = join '.', unpack('C4',$addrs[0]);
    $ip =~ /\d+\.\d+\.\d+\.\d+/ or die "FATAL iurt: could not resolve $host ip address";
    $run{chrooted_urpmi} =~ s/$host/$ip/;
    $run{chrooted_media} = "$run{chrooted_urpmi}/$run{distro}/$run{my_arch}";
    print {$run{LOG}} "iurt: using $run{chrooted_media} as insallation media\n" if $run{verbose} > 2
} else {
    $urpmi_options .= " --use-distrib $config->{repository}/$run{distro}/$run{my_arch}"
}

if (!$run{chrooted_urpmi} && $run{group}) {
    die "FATAL iurt: option --chrooted-urpmi is mandatory if --group is selected"
} 

my %config_usage = ( 
    admin => { desc => 'Mail of the administrator of packages builds', default => '' },
    all_media => { desc => 'List of known media', default => { 'main' => [ '' ], 'contrib' => [ '' ] } },
    basesystem_media_root => { desc => 'Name of the media holding basesystem packages', default => sub { my ($config, $run) = @_; "$config->{repository}/$run{distro}/$run{my_arch}/" } },
    basesystem_media => { desc => 'Where to find basesystem packages', default => 'main/release' },
    cache_home => { desc => 'Where to store the cache files', default => "$HOME/.bugs" },
    cache_min_size => { desc => 'Minimal size to consider a cache file valid', default => 1000000 },
    distribution => { desc => 'Name of the packages distribution', default => 'Mandriva Linux' },
    home => { desc => 'Home dir', default => $HOME },
    install_chroot_binary => { desc => 'Tool used to create initial chroot', default => 'install-chroot-tar.sh' },
    local_home => { desc => 'Where to build packages', default => $HOME },
    local_upload => { desc => 'Where to store build packages and log', default => '' },
    log_size_limit => { desc => 'Maximum authorized size for a log file', default => '100M' },
    log_size_date => { desc => 'Number of days log should be kept', default => '30' },
    log_url => { desc => 'Where the log can be seen', default => '' },
    minimum_package_number => { "Minimum number of packages in a synthesis file to consider it valid", default => 1000 },
    max_command_retry => { "Maximum number of retry Iurt will perform for a given command", fault => 20 },
    no_mail  => { desc => 'Hash table with people mail address where we should not send any mails', default => {} },
    packager => { desc => 'Name of the build bot', default => 'Iurt' },
    repository => { desc => 'Prefix of the repositories', default => '/mnt/BIG/dis/' },
    rsync_to => { desc => 'Server where the result of the builds should be rsynced (name@server:path format)', default => ''},
    sendmail => { desc => 'If the bot will send mail reports regarding build', default => 0 },
    supported_arch => { desc => 'Table of supported architecture', default => ['i586', 'x86_64'] },
    upload => { desc => 'Where to copy build packages', default => "$HOME/uploads/" },
    vendor => { desc => 'Name of the packages vendor', default => 'Mandriva' },
   );
config_usage() if $run{config_usage};
$run{my_arch} or usage($program_name, \@params, "no architecture given (media $run{media}, run{my_arch} $run{my_arch}, todo @{$run{todo}})");
if (!$arch_comp{$real_arch}{$run{my_arch}}) {
    die "FATAL iurt: could not compile $run{my_arch} binaries on a $real_arch"
}
foreach my $k (keys %config_usage) {
    ref $config_usage{$k}{default} eq 'CODE' and next;
    $config->{$k} ||= $run{config}{$k} || $config_usage{$k}{default}
}
foreach my $k (keys %config_usage) {
    ref $config_usage{$k}{default} eq 'CODE' or next;
    my $a = $config_usage{$k}{default}($config, \%run);
    $config->{$k} ||= $run{config}{$k} || $a
}

$config->{upload} .= $run{distro};
$config->{upload} =~ s/community//g;
if ($run{distro} ne 'cooker') {
    if ($run{media} ne 'main') {
	$config->{upload} .= "/$run{media}"
    }
} elsif ($run{media} eq 'contrib') {
    $config->{upload} =~ s/cooker/contrib/g;
}

-d $config->{upload} or usage($program_name, \@params, "$config->{upload} does not exist");

# cache file name is needed early to remove the manual lock file if the lock mechanism does not work
my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$run{media}.cache";
$run{cachefile} = $cachefile;
if (!$run{debug} && $run{media} || $run{chroot}) {
    $run{pidfile_home} = "$config->{cache_home}/";
    $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$run{media}";
    check_pid(\%run)
}

my $debug_tag = '_debug' if $run{debug};
my $chroot_name = "chroot_$run{distro_tag}$debug_tag";
my $chroot = "$config->{local_home}/$chroot_name";
$run{chroot_path} = $chroot;
my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
$run{chroot_tar} = $chroot_tar;
if ($run{chroot} || !-d "$chroot/dev") {
    check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot"
}

my $cache;
my $clear_cache = 1;
if (-f $cachefile && $run{use_cache}) {
    print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
    $cache = do $cachefile or print "FATAL iurt: could not load cache $cachefile ($!)\n";
    if (!$cache) {
	opendir my $cache_dir, $config->{cache_home};
	my $to_load;
	foreach my $file (readdir $cache_dir) {
	    (my $date) = $file =~ /iurt\.$run{distro_tag}\.$run{my_arch}\.$run{media}\.cache\.tmp\.(\d{8})/ or next;
	    if ($date > $to_load && -s "$config->{cache_home}/$file" > $config->{cache_min_size}) {
		$to_load = $date;
		$cachefile = "$config->{cache_home}/$file"
	    }
	}
	print {$run{LOG}} "iurt: loading alternate cache file $cachefile\n";
	$cache = do $cachefile or print "FATAL iurt: could not load cache $cachefile ($!)\n"
    }
    $clear_cache = 0 if $cache
}

if ($clear_cache) {
    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
}
$run{cache} = $cache;

my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint);
$to_compile = @{$run{todo}};
$to_compile += check_media(\%run, $cache, $config, \%srpm_version, \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media};
$to_compile += search_packages(1, $cache, \%provides, \%run, \%maint, \%srpm_version, @{$run{extra_dir}}) if $run{extra};

dump_cache(\%run);

print {$run{LOG}} "iurt: will try to compile $to_compile packages\n" if $run{verbose} > 1;

my ($fulldate, $daydate) = get_date();
if ($run{use_cache}) {
    $run{run} = $cache->{run};
    $run{run} ||= 1;
    $cache->{run} = $run{run} + 1
} else {
    $run{run} = "0.$fulldate"
}
print {$run{LOG}} "iurt: using $run{run} as chroot extension\n" if $run{verbose} > 4;
$run{user} = $ENV{SUDO_USER} || $ENV{USER};
$run{uid} = getpwnam $run{user};
print {$run{LOG}} "iurt: using local user $run{user}, id $run{uid}\n" if $run{verbose} > 3;
my $luser = $run{user} || 'builder';

my $unionfs_dir;
if ($run{unionfs}) {
    # FIXME need to grep /proc/modules not ot try to load it if already loaded
    open my $modules, '/proc/modules';
    my $ok;
    while (my $m = <$modules>) {
	if ($m =~ /unionfs/) {
	    $ok = 1;
	    last
	}
    }
    if (!$ok) {
	print {$run{LOG}} "iurt: adding unionfs module\n" if $run{verbose} > 0;
	system("sudo /sbin/depmod -a");
	system("sudo /sbin/modprobe -f unionfs") and $run{unionfs} = 0;
    }
    if ($run{unionfs}) {
	$unionfs_dir = "$config->{local_home}/iurt_unionfs$debug_tag/";
	remove_chroot(\%run, $unionfs_dir, \&clean_all_unionfs);
	$unionfs_dir = "$unionfs_dir/$run{user}/";
	-d $unionfs_dir or mkdir $unionfs_dir
    }
}

my (%done, $wait_limit, $done);
$run{done} = \%done;
my $home = $config->{local_home};
my $union_id = 1;
my $unionfs_tmp = $run{unionfs};	
my $chroot_tmp = "$config->{local_home}/chroot_tmp";
if (!-d $chroot_tmp) {
    mkdir $chroot_tmp
} else {
    remove_chroot(\%run, $chroot_tmp, \&clean_all_chroot_tmp, $chroot_name);
}
$chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}";
if (!-d $chroot_tmp) {
    mkdir $chroot_tmp
}
$chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}/$chroot_name.$run{run}";

# now exit if there is nothing to do and it was just a cleaning pass
if ($run{no_compile} || !@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild}) {
    send_status_mail(\%run, $config, $cache) if ($run{status_mail});
    print {$run{LOG}} "iurt: no package to compile :(\n";
    unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
    exit
}
print {$run{LOG}} "iurt: running with pid $$\n";

my $df = df $home;
if ($df->{per} == 100) {
    die "FATAL iurt: not enough space on the filesystem, only $df->{bavail} KB on $home, full at $df->{per}%"
}

if ($run{shell}) {
    ($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or die "FATAL iurt: could not create temporary chroot";
    add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL iurt: could not add local user";
    $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp";
    add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi') or die "FATAL iurt: could not add urpmi and sudo in the chroot";
    add_sudoers(\%run, $chroot_tmp, $luser);
    if ($run{shell}) {
	print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n";
	exec "sudo chroot $chroot_tmp /bin/su $luser -c bash";
	die "FATAL iurt: could not exec chroot to $chroot_tmp ($!)"
    }
}

$config->{local_upload} ||= $config->{local_home};
my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/";
if (!-d $local_spool) {
    print {$run{LOG}} "iurt: creating local spool $local_spool\n" if $run{verbose} > 4;
    mkdir_p("$local_spool/log") or die "FATAL iurt: could not create local spool dir $local_spool ($!)"
}

# perform some cleaning before running to have some more space, rsync to the server too in case previous iurt crashed
if ($config->{rsync_to} && !$run{no_rsync}) { 
    	# remove some old and very big log files not to saturate the server
	system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|);
	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/");
}

print {$run{LOG}} "iurt: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3;
if (!dump_rpmmacros("$chroot/home/builder/.rpmmacros")) {
     print {$run{LOG}} "ERROR iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros, trying to rebuild the chroot\n";
     check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot";
     dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros"
}

my $s = sub { 
    if ($run{main}) {
	print {$run{LOG}} "iurt: dumping cache...\n"; 
	dump_cache(\%run);
	$Data::Dumper::Indent = 0;
	$Data::Dumper::Terse = 1;
	print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n" if $run{verbose} > 5; 
	print {$run{LOG}} "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n";
    }
    exit 
};
$SIG{TERM} = $s;
$SIG{INT} = $s;
$run{main} = 1;

my $local_media;
my $rebuild;
if ($run{group}) { 
    $rebuild = 1;
    order_packages(\%run, $config, $union_id) or die "FATAL iurt: could not order packages";
    $local_media = "$local_spool/$run{run}/"
}
do { 
    foreach (my $i ; $i < @{$run{todo}}; $i++) {
	my ($dir, $srpm, $status) = @{$run{todo}[$i]};
	$status or next;
	$done{$srpm} and next;
	$done{$srpm} = 1;
	check_version($srpm, \%srpm_version) or next;
	if ($run{debug}) { $run{debug}++ == 2 and exit }
	$done++;
	mkdir "$local_spool/log/$srpm";
	print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n";
	# FIXME unfortunately urpmi stalls quite often
	my $retry = 0;
	# 
	# current rpm is sometime segfaulting, and iurt is them blocked and cannot 
	#
	# $cache->{failure}{$srpm} = 1;
	# dump_cache(\%run);
	retry:
	if (!$run{chrooted_urpmi}) {
	    my $match = "urpmi $urpmi_options --root $chroot_tmp";
	    if (!clean_process($match, $run{verbose})) {
		dump_cache(\%run);
		die "FATAL iurt: Could not have urpmi working !"
	    }
	}
	my ($u_id, $unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm) or next;
	$union_id = $u_id;

	if ($run{chrooted_urpmi}) { 
	    $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp ";
	    add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi') or next;
	    add_media(\%run, $config, $chroot_tmp, 'Main', "--distrib $run{chrooted_media}") or next;
	    if (-d "$local_media/hdlist.cz") {
		mkdir("$chroot_tmp/iurt_media/");
		opendir my $dir, $local_media;
		my $next;
		foreach my $f (readdir $dir) {
		    $f =~ /(\.rpm|^hdlist.cz)$/ or next;
		    if (!link "$local_media/$f", "$chroot_tmp/iurt_media") {
			if (!copy "$local_media/$f", "$chroot_tmp/iurt_media") {
			    print {$run{LOG}} "ERROR iurt: could not copy file $local_media/$f to $chroot_tmp/iurt_media";
			    $next = 1;
			    last
			}
		    }
		}
		next if $next;
		add_media(\%run, $config, $chroot_tmp, 'iurt_group', "iurt_group file:///iurt_media") or next
	    }
	    $run{urpmi_command} = "chroot $chroot_tmp urpmi $urpmi_options "
	} else {
	    $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp"
	}
	my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
	my ($maintainer, $cc);
	if (!$run{warn}) {
	    ($maintainer) = get_maint(\%run, $srpm);
	    $cc = "$maint{$srpm}";#, maintainers\@mandriva.com";
	    chomp $maintainer;
	    if (!$maintainer || $maintainer eq 'NOT_FOUND') {
		$maintainer = $cc;
		#$cc = 'maintainers@mandriva.com'
	    }
	}
	#($maintainer, $cc) = ($config->{admin},'');

	print {$run{LOG}} "iurt: adding local user $luser into $chroot_tmp...\n" if $run{verbose};
	add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or next;

	my $ret = recreate_srpm(\%run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry);
	if ($ret == -1) {
	    $retry = 1;
	    goto retry
	} elsif (!$ret) {
	    next
	} 
	
	print {$run{LOG}} "iurt: installing build dependencies of $srpm...\n" if $run{verbose};
	my $wait_urpmi = sub { 
	    print {$run{LOG}} "WARNING iurt: urpmi database locked, waiting...\n" if $run{debug};
	    sleep 30; 
	    $wait_limit++; 
	    if ($wait_limit > 8) { 
		$wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) 
	    } };
	my $path_srpm =  $run{chrooted_urpmi} ? "/home/$luser/rpm/SRPMS/" : "$chroot_tmp/home/$luser/rpm/SRPMS/";
	# 
	# on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore 
	# installed packages, retrying several time to be sure something is really broken
	my $ok;
	foreach my $try ([ '', 'Trying basic uprmi command' ], [ '', 'Rebuilding the rpm db and trying againg', '_retry' ], [ ' --allow-nodeps', 'trying harder with --allow-nodeps' , '_nodeps'], [ ' --no-install', 'try the hard way, with rpm', '_rpm']) {
	    my ($opt, $msg, $suf) = @$try;
	    print {$run{LOG}} "iurt: $msg\n" if $run{verbose};
	    my $unsatisfied;
	    if (!perform_command("sudo $run{urpmi_command} $opt $path_srpm/$srpm", 
		\%run, $config,
		error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}",  
		hash => "install_deps_$srpm$suf", 
		timeout => 600, 
		srpm => $srpm,
		freq => 1,
		#cc => $cc, 
		retry => 3,
		debug_mail => $run{debug},
		error_regexp => 'cannot be installed',
		wait_regexp => { 
		'database locked' => $wait_urpmi, 
		'is needed by' => sub { 
		    print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; 
		    system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null");
		    1
		    }, 
	         }, 
		 log => "$local_spool/log/$srpm/",
		 callback => sub { 
			my ($opt, $output) = @_;
			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
# 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error).	    
# my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g;
			my @missing_deps = $output =~ /([^ \n]+) \(due to unsatisfied ([^[ \n]*)(?: ([^\n]*)|\[([^\n]*)\])?\)/g;
			# as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
			@missing_deps or return 1;
			$unsatisfied = 1;
			while (my $missing_package = shift @missing_deps) {
			    my $missing_deps = shift @missing_deps;
			    my $version = shift @missing_deps;
			    my $version2 = shift @missing_deps;
			    $version ||= $version2 || 0;
			    my $p = $pack_provide{$missing_deps} || $missing_deps;
			    my ($missing_package_name, $first_maint);
			    if ($missing_package !~ /\.src$/) {
				($first_maint, $missing_package_name) = get_maint(\%run, $missing_package);
				print {$run{LOG}} "iurt: likely $missing_package_name need to be rebuilt ($first_maint)\n" if $run{verbose} > 4;
			    } else {
				$missing_package = ''
			    }
			    my ($other_maint) = get_maint(\%run, $p);
			    print {$run{LOG}} "missing dep: $missing_deps ($other_maint) missing_package $missing_package ($first_maint)\n";
			    foreach my $m ($first_maint, $other_maint) {
				if ($other_maint && $other_maint ne 'NOT_FOUND') {
				    $opt->{mail} = $config->{admin};
				    #$opt->{mail} .= ", $other_maint";
				}
			    }
			    if (!$opt->{mail}) { 
				$opt->{mail} = $config->{admin}
			    }
			    # remember what is needed, and do not try to recompile until it is available
			    if ($missing_package) {
				$opt->{error} = "[MISSING] $missing_deps, needed by $missing_package to build $srpm, is not available on $run{my_arch} (rebuild $missing_package?)";
				$cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $first_maint || $other_maint || $maintainer };
			    } else {
				$opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
				$cache->{needed}{$srpm}{$missing_deps} = { package => $missing_package , version => $version, maint => $maintainer || $other_maint };
			    }
			} 
			0
		    },
		)) {
		if (!clean_process("$run{urpmi_command} $opt $path_srpm/$srpm", $run{verbose})) {
		    dump_cache(\%run);
		    die "FATAL iurt: Could not have urpmi working !"
		}
		$unsatisfied and last
	    } else {
		$ok = 1;
	    }
	    if (!$ok && (system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db*") || system("sudo chroot $chroot_tmp rpm --rebuilddb"))) {
		print {$run{LOG}} "ERROR iurt: rebuilding rpm db failed, aborting ($!)\n";
		last
	    }
	    if ($suf eq '_rpm') {
		print {$run{LOG}} "iurt: trying to install all the rpms in $chroot_tmp/var/cache/urpmi/rpms/ manually\n" if $run{verbose}; 
		if (!system("sudo chroot $chroot_tmp rpm -Uvh --force --nodeps /var/cache/urpmi/rpms/*.rpm")) {
		    $ok = 1;
		    last
		} else {
		    $ok = 0
		}
	    }
	    last if $ok == 1
	}
	if (!$ok) {
	    $run{status}{$srpm} = 'install_deps_failure';
	    next;
	}
	# try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory
	# running rpm -qa several time seems to fix the problem
	system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null");
	perform_command("sudo chroot $chroot_tmp rpm -qa", 
	    \%run, $config,
	    hash => "rpm_qa_$srpm", 
	    timeout => 60, 
	    debug_mail => $run{debug},
	    log => "$local_spool/log/$srpm/"); # or next; As this failed quite often, do not stop
	print {$run{LOG}} "Compiling $srpm\n" if $run{verbose}; 
	my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm";
	if ($run{stop}) {
	    add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi', 'sudo');
	    add_sudoers(\%run, $chroot_tmp, $luser);
	    $command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec"
	}
	if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, 
		\%run, $config,
		#	mail => $maintainer, 
		error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", 
		hash => "build_$srpm", 
		timeout => 18000, 
		srpm => $srpm,
		debug_mail => $run{debug},
		cc => $cc, 
		log => "$local_spool/log/$srpm/", 
		error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
		callback => sub { 
		    my ($opt, $output) = @_;
		    if ($run{stop}) {
			print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp (pid $$)\n";
			# exec does not work because it seems stdin and out are shared between children
			system("sudo chroot $chroot_tmp /bin/su $luser -c bash");
			exit
		    }
		    print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{verbose} > 4;
		    if ($unionfs_tmp && $output =~ /no space left on device/i) {
			print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
			$cache->{no_unionfs}{$srpm} = 1;
			return 1
		    } elsif ($unionfs_tmp && $output =~ m,$home,) {
			print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n";
			$cache->{no_unionfs}{$srpm} = 1;
			return 1
		    } elsif ($output =~ /bin\/ld: cannot find -l(\S*)|configure.*error.* (?:-l([^\s]+)|([^\s]+) includes)/) {
			my $missing = $1;
			my @rpm = find_provides(\%run, \%pack_provide, $missing);
			print {$run{LOG}} "iurt: likely @rpm ($missing-devel) needed to rebuilt $srpm is not in build_requires\n" if $run{verbose} > 4;
			if ($maintainer ne 'NOT_FOUND') {
			    $opt->{mail} = $config->{admin};
			    #$opt->{mail} .= ", $other_maint";
			}
			if (!$opt->{mail}) { 
			    $opt->{mail} = $config->{admin};
			}
			if (@rpm > 1) {
			    $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] one of @rpm ($missing-devel), needed to build $srpm, is not in buildrequires";
			} elsif (@rpm == 1) {
			    $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] @rpm ($missing-devel), needed to build $srpm, is not in buildrequires";
			} else {
			    $opt->{error} = "[MISSING_BUILD_REQUIRES_TAG] $missing-devel, needed to build $srpm, is not in buildrequires";
			}
			$cache->{buildrequires}{$srpm}{$missing} = \@rpm;
			return 
		    }
		    1
		}, 
		freq => 1)) {
	    # FIXME
	    # The simple algo used here is : 
	    #  try to compile it with unionfs, if it runs out of space, compile it without the next time
	    #
	    #  This could be improved in keeping this srpm name for future version, but if we compile it 
	    #  on a new machine with more ram, or if next version compiles just fine with unionfs, we will
	    #  loose the unionfs advantage. 
	    #
	    #  Maybe the right thing to do would be to first try to increase the tmpfs size (more than 50 % of the 
	    #  physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs
	    #  plus). Or to keep the faulty package a unionfs exception for some time, to save some more extra builds.
	    # 
	    if (!glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") {
		if ($unionfs_tmp && $cache->{no_unionfs}{$srpm}) {
		    goto retry
		}
		$cache->{failure}{$srpm} = 1;
		$run{status}{$srpm} = 'build_failure';
		# 20060615
		dump_cache(\%run);
		dump_status($local_spool, \%run);
		next
	    }
	}
	# do some cleaning if the compilation is successful
	delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm};
	delete $cache->{buildrequires}{$srpm} if defined $cache->{buildrequires}{$srpm};
	if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", 
		\%run, $config,
		#	mail => $maintainer, 
		error => "[REBUILD] binaries packages generated from $srpm do not install correctly", 
		hash => "binary_test_$srpm", 
		srpm => $srpm,
		timeout => 600, 
		debug_mail => $run{debug},
		freq => 1, 
		wait_regexp => { 'database locked' => $wait_urpmi },
		error_regexp => 'unable to access',
		log => "$local_spool/log/$srpm/")) {
	    $cache->{failure}{$srpm} = 1;
	    $run{status}{$srpm} = 'binary_test_failure';
	    next
	}
	$run{status}{$srpm} = 'ok';
	delete $cache->{failure}{$srpm} if defined $cache->{failure}{$srpm};
	if ($run{debug}) {
	    print {$run{LOG}} "iurt: debug mode, skip other packages\n";
	    exit
	} elsif ($run{group}) {
	    print {$run{LOG}} "iurt: group mode, keep packages for local media\n";
	    $run{done}{$srpm} = $done;
	    system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_media ($!)\n";
	    system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_media ($!)\n";
	} else {
	    print {$run{LOG}} "iurt: build successful, copying packages to $local_spool.\n";
	    system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_spool ($!)\n";
	    if ($run{copy_srpm}) {
		system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_spool ($!)\n";
	    }
	    process_queue($config, \%run, \@wrong_rpm, 1)
	}
	# dymp_cache each time so that concurrent process can get updated
	dump_cache(\%run) if $run{concurrent_run};
    }
    if ($run{group}) {
	$rebuild = 1 if order_packages(\%run, $config, $union_id)
    }
} while ($rebuild);

print {$run{LOG}} "iurt: reprocess generated packages queue\n" if $run{verbose};
process_queue($config, \%run, \@wrong_rpm);

dump_cache(\%run);

print {$run{LOG}} "ERROR iurt: RPM with a wrong SRPM name\n" if @wrong_rpm;
if (open my $file, ">$local_spool/log/wrong_srpm_names.log") {
    foreach (@wrong_rpm) {
	print $file "$_->[1] -> $_->[0] (", $cache->{rpm_srpm}{$_->[1]},")\n";
    }
    close $file
}

dump_status($local_spool, \%run);

send_status_mail(\%run, $config, $cache) if ($run{status_mail});

if ($config->{rsync_to} && !$run{no_rsync}) { 
    	# remove some old and very big log files not to saturate the server
	system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|);
	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/");
}

# one last try to clean
print {$run{LOG}} "iurt: try to clean remaining unionfs\n" if $run{verbose};
if ($run{unionfs}) {
    my ($dir) = $unionfs_dir =~ /(.*)\/[^\/]+\/?/;
    remove_chroot(\%run, $dir, \&clean_all_unionfs)
}
unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
exit;

sub config_usage {
	print "
	
	Iurt configuration keywords:
	
";
    $Data::Dumper::Indent = 0;
    $Data::Dumper::Terse = 1;
    foreach my $k (sort keys %config_usage) {
	print "	$k: $config_usage{$k}{desc} 
			default: ", Data::Dumper->Dump([$config_usage{$k}{default}]), ", current: ", Data::Dumper->Dump([$config->{$k}]),"\n"
    }
    print "\n\n";
    exit
}

sub clean_all_unionfs {
    my ($unionfs_dir) = @_;
    print {$run{LOG}} "Cleaning old unionfs remaining dir in $unionfs_dir\n" if $run{verbose} > 1;
    my $dir;
    if (!opendir $dir, $unionfs_dir) {
	print {$run{LOG}} "FATAL iurt: could not open $unionfs_dir ($!)";
	return
    }
    foreach (readdir $dir) {
	/unionfs\.((?:0\.)?\d+)\.(\d*)$/ or next;
	clean_unionfs($unionfs_dir, \%run, $1, $2);
    }
    closedir $dir
}

sub clean_all_chroot_tmp {
    my ($chroot_dir, $prefix) = @_;
    print {$run{LOG}} "Cleaning old chroot remaining dir in $chroot_dir\n" if $run{verbose};
    my $dir;
    if (!opendir $dir, $chroot_dir) { 
	print "ERROR iurt: could not open $chroot_dir ($!)";
	return
    }
    foreach (readdir $dir) {
	/$prefix.*$/ or next;
	clean_chroot_tmp($chroot_dir, $_);
    }
    closedir $dir
}

sub clean_chroot {
    my ($chroot, $run, $only_clean) = @_;
    if (-d $chroot) {
	system("sudo umount $chroot/proc &> /dev/null");
	system("sudo umount $chroot/dev/pts &> /dev/null");
	perform_command("sudo rm -rf $chroot", 
	    $run, $config,
	    mail => $config->{admin},
	    error => "[REBUILD] Deleting of old chroot $chroot failed", 
	    hash => 'chroot_deletion', 
	    debug_mail => $run->{debug},
	    die => 1);
    }
   return 1 if $only_clean;
   mkdir $chroot;
   perform_command("pushd $chroot && sudo tar xvf $chroot_tar", 
		$run, $config,
		mail => $config->{admin}, 
		error => "[REBUILD] creating the initial chroot $chroot failed", 
		hash => 'chroot_init', 
		debug_mail => $run->{debug},
		die => 1);

   dump_rpmmacros("$chroot/home/builder/.rpmmacros") or return;
   system("sudo mount none -t proc $chroot/proc &>/dev/null") and return;
   system("sudo mount none -t devpts $chroot/dev/pts &>/dev/null") and return;
   1
}

sub clean_process {
    my ($match, $verbose) = @_;
    return clean($match, "pgrep -u root -f", "sudo pkill -9 -u root -f", $verbose);
}

sub clean_mnt {
    my ($mount_point, $verbose) = @_;
    return clean($mount_point, "/sbin/fuser", "sudo /sbin/fuser -k", $verbose);
}

sub clean {
    my ($var, $cmd, $kill_cmd, $verbose) = @_;
    my $ps;
    my $i;
    use POSIX ":sys_wait_h";
    while ($ps = `$cmd "$var"`) {
	system(qq{$kill_cmd "$var" &>/dev/null});
	sleep 1;
	$ps =~ s/\n/,/g;
	print {$run{LOG}} "Trying to remove previous blocked processes for $var ($ps)\n" if $verbose > 1;
	waitpid(-1, WNOHANG);
	return 0 if $i++ > 10
    }
    1
}

sub clean_unionfs {
    my ($unionfs_dir, $run, $r, $union_id) = @_;
    -d "$unionfs_dir/unionfs.$r.$union_id" or return $union_id;
    print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$r.$union_id\n" if $run->{verbose} > 1;
    my $nok = 1;
    my $path = "$unionfs_dir/unionfs.$r.$union_id";
    while ($nok) {
	$nok = 0;
	foreach my $fs ([ 'proc', 'proc'],[ 'dev/pts', 'devpts']) {
	    my ($dir, $type) = @$fs;
	    if (-d "$path/$dir" && check_mounted("$path/$dir", $type)) {
		print {$run->{LOG}} "iurt iurt: umounting $path/$dir\n" if $run->{verbose} > 2;
		if (system("sudo umount $path/$dir &>/dev/null")) { 
		    print {$run->{LOG}} "ERROR iurt: could not umount $path/$dir\n";
		    return $union_id + 1
		}
	    }
	}
	foreach my $t ("unionfs",'tmpfs') {
	    # unfortunately quite oftem the unionfs is busy and could not be unmounted
	    my $d = "$unionfs_dir/$t.$r.$union_id";
	    my $last;
	    if (-d $d && check_mounted($d, $t)) {
		$nok = 1;
		system("sudo /sbin/fuser -k $d &> /dev/null");
		print {$run->{LOG}} "iurt: umounting $d\n" if $run->{verbose} > 2;
		if (system(qq{sudo umount $d &> /dev/null})) {
		    print {$run->{LOG}} "WARNING iurt: could not umount $d ($!)\n" if $run->{verbose} > 1;
		    return $union_id + 1;
		}
	    }
	}
    }
    foreach my $t ("unionfs",'tmpfs') {
	my $d = "$unionfs_dir/$t.$r.$union_id";
	print {$run->{LOG}} "iurt: removing $d\n" if $run->{verbose} > 1;
	if (system(qq{sudo rm -rf $d})) {
	    print {$run->{LOG}} "ERROR iurt: removing $d failed ($!)\n";
	    return $union_id + 1
	}
    }
    $union_id
}

sub clean_chroot_tmp {
    my ($chroot_dir, $dir) = @_;
    my $d = "$chroot_dir/$dir";
    foreach my $m ( 'proc', 'dev/pts') {
	if (system("sudo umount $d/$m &>/dev/null") && $run{verbose} > 1) { 
	    print {$run{LOG}} "ERROR iurt: could not umount /$m in $d/\n" 
	}
    }
    print {$run{LOG}} "iurt: cleaning $d\n" if $run{verbose};
    system("sudo /sbin/fuser -k $d &> /dev/null");
    print {$run{LOG}} "iurt: removing $d\n" if $run{verbose};
    system(qq{sudo rm -rf $d});
}

sub check_mounted {
    my ($mount_point, $type) = @_;
    my $mount;
    if (!open $mount, '/proc/mounts') {
	print 'ERROR iurt: could not open /proc/mounts';
	return
    }
    $mount_point =~ s,//+,/,g;
    while (<$mount>) {
	return 1 if /^\w+ $mount_point $type /
    }
    0
}

sub check_needed {
    my ($srpm, $cache, $provides) = @_;
    if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 } 
    my $ok = 1;
    # migrate old cache format
    my $ent = $cache->{needed}{$srpm};
    if (ref $ent eq 'ARRAY') {
	my $table = $ent;
	$cache->{needed}{$srpm} = {};
	foreach my $t (@$table) {
	    my ($missing, $version, $maint) = @$t;
	    $cache->{needed}{$srpm}{$missing} = {
		version => $version,
		maint => $maint
	    }
	}
	$ent = $cache->{needed}{$srpm}
    }
    foreach my $name (keys %$ent) {
	my ($package, $version, $maint) = @{$ent->{$name}}{'package', 'version','maint'};
	# if packages does not exist anymore, it may have been rebuild, then try to recompute the build dependencies
	last if $package && !$provides->{$package};
	my ($p_version) = $provides->{$name};
	if ($p_version) {
	    next if $version == $p_version;
	    next if URPM::ranges_overlap($version, $p_version)
	}
	$ok = 0;
	if ($version) {
	    $ent->{$name}{version} = $version;
	}
	my $v ||= $version;
	if ($package) {
	    print {$run{LOG}} "ERROR iurt: $srpm needs package $package which requires missing $name $v to be compiled.\n";
	} else {
	    print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n";
	} 
	# try to recompile it once in a while
	last if $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72;
	return 1
    }
    delete $cache->{needed}{$srpm} if $ok;
    $ok
}

sub process_queue {
    my ($config, $run, $wrong_rpm, $quiet) = @_;
    return if !$run->{upload} && $quiet;
    my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}/$run->{media}/";
    opendir my $rpmdir, $dir or return;
    foreach my $rpm (readdir $rpmdir) {
	my ($rarch, $srpm) = update_srpm($dir, $rpm, $wrong_rpm);
	$rarch or next;
	print {$run{LOG}} "iurt: $rpm\n";
	next if !$run->{upload};
	# recheck if the package has not been uploaded in the meantime
	my $rpms_dir = "$config->{repository}/$run->{distro}/$run->{my_arch}/media/$run->{media}/";
	if (! -f "$rpms_dir/$rpm") {
	    my $ok = copy "$dir/$rpm", "$config->{upload}/$config->{extra_subdir}/RPMS/";
	    # try to keep the opportunity to prevent disk full
	    if (!$ok){
		print {$run{LOG}} "ERROR process_queue: cannot copy $dir/$rpm to $config->{upload}/$config->{extra_subdir}/RPMS/ ($!)\n";
		next
	    }
	}
	if ($run->{upload_source}) {

	}
	unlink "$dir/$rpm";
	$cache->{queue}{$srpm} = 1
    }
    closedir $rpmdir;
}

sub update_srpm {
	my ($dir, $rpm, $wrong_rpm) = @_;
	my ($arch) = $rpm =~ /([^\.]+)\.rpm$/ or return 0;
	my $srpm = $cache->{rpm_srpm}{$rpm};
	if (!$srpm) {
		my $hdr = RPM4::Header->new("$dir/$rpm");
		$hdr or return 0;
		$srpm = $hdr->queryformat('%{SOURCERPM}');
		$cache->{rpm_srpm}{$rpm} = $srpm
	}
	$srpm = fix_srpm_name($srpm, $rpm, $wrong_rpm);
	$arch, $srpm
}

sub dump_cache {
    my ($run) = @_;
    my $filename = $run->{cachefile};
    my $cache = $run->{cache};
    # Right now there are no mechanism of concurrent access/write to the cache. There is 
    # on global lock for one iurt session. A finer cache access would allow several iurt running
    # but the idea is more to have a global parrallel build than several local ones.
    return if $run->{debug} || !$run->{use_cache};
    open my $file, ">$filename.tmp.$daydate" or die "FATAL iurt dump_cache: cannot open $filename.tmp";
    if ($run{concurrent_run}) {
	print {$run{log}} "iurt: merging cache";
	my $old_cache;
	if (-f $filename) {
	    print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
	    $old_cache = do $cachefile;
	    foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning', 'buildrequires') {
		foreach my $rpm (%{$old_cache->{$k}}) {
		    $cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm}
		}
	    }
	}
	#  $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
    }
    $Data::Dumper::Indent = 1;
    $Data::Dumper::Terse = 1;
    print $file Data::Dumper->Dump([ $cache ], [ "cache" ]);
    # flock does not work on network files and lockf seems to fail too
    my $status = 1; #File::lockf::lock($file);
    if (!$status) {
	unlink $filename;
	link "$filename.tmp.$daydate", $filename;
	File::lockf::ulock($file)
    } else {
	print {$run{LOG}} "WARNING iurt: locking the cache file $cachefile failed (status $status $!), try to lock manually\n";
	if (-f "$filename.lock") {
	    print {$run{LOG}} "ERROR iurt: manual file lock exist, do not save the cache\n";
	} else {
	    open my $lock, ">$filename.lock";
	    print $lock $$;
	    close $lock;
	    unlink $filename;
	    link "$filename.tmp.$daydate", $filename;
	    unlink "$filename.lock"
	}
    }
}

sub sendmail {
	my ($to, $cc, $subject, $text, $from, $debug) = @_;
	do { print "Cannot find sender-email-address [$to]\n"; return } unless defined($to);
	$from ||= $config->{packager};
	my $MAIL;
	if (!$debug) { open $MAIL,  "| /usr/sbin/sendmail -t" or return } else { open $MAIL,  ">&STDOUT" or return }
	my $sender = encode_mimewords($to);
	my $subject = encode_mimewords($subject);
	print $MAIL "To: $sender\n";
	if ($cc) { $cc = encode_mimewords($cc); print $MAIL "Cc: $cc\n" }
	print $MAIL "From: $from\n";
	print $MAIL "Subject: $subject\n";
	print $MAIL "\n";
	print $MAIL $text; 
	close($MAIL)
}

sub check_arch {
	my ($hdr) = @_;
	my (@exclusive_arch) = $hdr->queryformat('%{EXCLUSIVEARCH}');
	return if ! grep { $arch_comp{$run{my_arch}}{$_} || $_ eq '(none)' } @exclusive_arch;
	my (@exclude_arch) = $hdr->queryformat('%{EXCLUDEARCH}');
	return if grep { $_ eq $run{my_arch} } @exclude_arch;
	1
}

sub check_version {
	my ($srpm, $srpm_version) = @_;
	my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm/;
	if (URPM::ranges_overlap("= $srpm",">= $srpm_version->{$srpm_name}")) {
		$srpm_version->{$srpm_name} = $srpm;
		return 1
	}
	0
}

sub fix_srpm_name {
	my ($srpm, $rpm, $wrong_rpm) = @_;
	my $old_srpm = $srpm;
	if ($srpm =~ s/^lib64/lib/){
		push @$wrong_rpm, [ $old_srpm, $rpm ];
		$cache->{rpm_srpm}{$rpm} = $srpm
	}
	$srpm
}

sub perform_command {
    my ($command, $run, $config, %opt) = @_;
    $opt{timeout} ||= 300;
    $opt{freq} ||= 24;
    print {$run{LOG}} "Timeout $opt{timeout}\n" if $run{verbose} > 2;
    # from alarm perldoc
    my ($output, $fulloutput, $comment);
    my ($kill, $pipe);
    if ($opt{debug}) {
	print "Would have rum $command with a timeout of $opt{timeout}\n";
	return 1
    }
    local $SIG{PIPE} = sub { print "Broken pipe!\n"; $pipe = 1 };
    my $retry = $opt{retry} || 1;
    my $call_ret = 1;
    my ($err, $pid, $try);
    my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log";
    my $max_retry = $config->{max_command_retry} < $retry ? $retry : $config->{max_command_retry};
    while ($retry) {
	$try++;
	if ($opt{retry} > 1) {
	    $logfile = "$opt{log}/$opt{hash}-$try.$run->{run}.log";
	}
	if ($opt{log}) {
	    my $parent_pid = $$;
	    $pid = fork;
	    #close STDIN; close STDERR;close STDOUT;
	    my $tot_time;
	    if (!$pid) {
		print {$run{LOG}} "Forking to monitor log size\n" if $run{verbose} > 2;
		$run->{main} = 0;
		local $SIG{ALRM} = sub { exit };
		$tot_time += sleep 30;
		my $size_limit = $config->{log_size_limit};
		$size_limit =~ s/k/000/i;
		$size_limit =~ s/M/000000/i;
		$size_limit =~ s/G/000000000/i;
		while ($tot_time < $opt{timeout}) {
		    my (@stat) = stat $logfile;
		    if ($stat[7] > $size_limit) {
			print {$run{LOG}} "WARNING: killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})\n";
			kill 14, "-$parent_pid";
			exit
		    }
		    my $df = df $opt{log} ;
		    if ($df->{per} == 100) {
			print {$run{LOG}} "WARNING: killing current command because running out of disk space (only $df->{bavail}KB left)\n";
			kill 14, "-$parent_pid";
			exit
		    }
		    $tot_time += sleep 30;
		}
		exit
	    }
	}
	eval {
	    local $SIG{ALRM} = sub { print "Timeout!\n"; $kill = 1; die "alarm\n" }; # NB: \n required
	    alarm $opt{timeout};
	    print {$run{LOG}} "$command\n" if $run{verbose} > 2;
	    if ($opt{log}) {
		#$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
		system("$command &> $logfile");
	    } else {
		$output = `$command 2>&1`;
	    }
	    alarm 0
	};
	$err = $?;
	# kill pid watching log file size
	if ($pid) {
	    kill_for_good($pid);
	}
	if ($@) {
	    # timed out
	    die "FATAL iurt: unexpected signal" unless $@ eq "alarm\n";   # propagate unexpected errors
	}
	# Keep the run first on the harddrive so that one can check the command status tailing it
	if ($opt{log} && open my $log, $logfile) {
	    local $/;
	    $output = <$log>
	}
	$fulloutput .= $output;
	if (ref $opt{callback}) {
	    $call_ret = $opt{callback}(\%opt, $output);
	    $call_ret == -1 and return 1;
	    $call_ret == -2 and return 0
	}
	if ($kill) {
	    $comment = "Command has been killed after $opt{timeout} seconds: $command\n";
	    my ($cmd_to_kill) = $command =~ /sudo(?: chroot \S+)? (.*)/;
	    clean_process($cmd_to_kill, $run{verbose})
	} elsif ($pipe) {
	    $comment = "Command receives a broken pipe: $command\n";
	    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: broken pipe", "$comment\n$output", 0, 0, $opt{debug_mail});
	} else {
	    $comment = "Command failed: $command\n"
	}
	# Maybe this has to be put before all the commands altering the $output var
	my $inc;
	if ($opt{wait_regexp}) {
	    foreach my $wr (keys %{$opt{wait_regexp}}) {
		if ($output =~ /$wr/m) {
		    $inc = $opt{wait_regexp}{$wr}(\%opt, $output) if ref $opt{wait_regexp}{$wr};
		    print {$run->{LOG}} "ERROR iurt: $wr !\n";
		    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$wr\n\n$comment\n$output", 0, 0, $opt{debug_mail}) if $opt{wait_mail}
		}
	    }
	}
	if ($inc && $try < $max_retry) {
	    $retry += $inc	    
	} elsif ($call_ret && !$kill && !$err && !$opt{error_regexp} || $fulloutput !~ /$opt{error_regexp}/) {
	    $retry = 0
	} else {
	    $retry-- 
	}
    }
    if (!$call_ret || $kill || $err || $opt{error_regexp} && $fulloutput =~ /$opt{error_regexp}/) {
	print {$run->{LOG}} "ERROR iurt: call_ret $call_ret kill $kill err $err ($opt{error_regexp})\n" if $run->{verbose} > 4;;
	if ($opt{log} && $config->{log_url}) {
	    $comment = qq|See $config->{log_url}/$run->{distro_tag}/$run->{my_arch}/$run->{media}/log/$opt{srpm}/\n\n$comment|
	}
	my $out;
	if (length $fulloutput < 10000) {
	    $out = $fulloutput
	} else { 
	    $out = "Message too big, see http link for details\n"
	}

	if ($opt{mail} && $config->{sendmail} && !$config->{no_mail}{$opt{mail}}) {
	    if (! ($cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq})) {
		my $cc = join ',', grep { !$config->{no_mail}{$_} } split ',', $opt{cc};
		sendmail($opt{mail}, $cc,  $opt{error} , "$comment\n$out", 0, 0, $opt{debug_mail});
	    } elsif ($config->{admin}) {
		sendmail($config->{admin}, '' , $opt{error}, "$comment\n$out", 0, 0, $opt{debug_mail});
	    }
	}
	$cache->{warning}{$opt{hash}}{$opt{mail}}++;
	print {$run->{LOG}} "\n$comment\n$fulloutput\n";
	if ($opt{die}) {
	    dump_cache($run);
	    die "FATAL iurt: $opt{error}."
	}
	return 0
    }
    1
}

sub kill_for_good {
    my ($pid) = @_;
    use POSIX ":sys_wait_h";
    kill 14, $pid;
    sleep 1;
    waitpid(-1, WNOHANG);
    if (getpgrp $pid != -1) {
	kill 15, $pid;
	sleep 1;
	waitpid(-1, WNOHANG);
	if (getpgrp $pid  != -1) {
	    print STDERR "WARNING iurt: have to kill -9 pid $pid\n";
	    kill 9, $pid;
	    sleep 1;
	    waitpid(-1, WNOHANG);
	}
    }
}

sub check_chroot {
    my ($chroot, $chroot_tar, $run, $opt) = @_;
    print {$run{LOG}} "iurt: checking basesystem tar\n" if $run{verbose};
    system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot" &> /dev/null});
    if (!clean_chroot($chroot, $run, 1)) {
	print {$run{LOG}} "ERROR iurt: Could no prepare initial chroot";
	return
    }
    my (@stat) = stat $chroot_tar;
    if (time - $stat[9] > 604800) {
	print {$run{LOG}} "iurt check_chroot: tar chroot is more than one week old, forcing the rebuild\n";
	system("sudo rm -rf $chroot_tar $chroot");
    }
    perform_command("sudo $config->{install_chroot_binary} $config->{basesystem_media_root} $config->{basesystem_media_root}/media/$config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build rpm-mandriva-setup-build", 
	$run, $config,
	mail => $config->{admin},
	error => "[REBUILD] Creating the inital chroot for $run->{distro_tag} on $run->{my_arch} failed", 
	hash => 'chroot_inititialization', 
	timeout => 600, 
	debug_mail => $run->{debug},
	die => 1);
}

sub dump_rpmmacros {
	my ($file) = @_;
	my $f;
	if (!open $f, qq{| sudo sh -c "cat > $file"}) {
	   print {$run{LOG}} "ERROR iurt: could not open $file ($!)\n";
	   return 0
	}
	print $f qq{\%_topdir                \%(echo \$HOME)/rpm
\%_tmppath               \%(echo \$HOME)/rpm/tmp/
\%distribution           $config->{distribution}
\%vendor                 $config->{vendor}
\%packager               $config->{packager}};
        close $f;
	-f $file or return 0;
	1
}

sub check_pid {
    my ($run) = @_;
    my $hostname = `hostname`;
    chomp $hostname;
    my $pidfile = $run->{pidfile};
    my $lockfile = "$run->{pidfile_home}/$pidfile.$hostname.pid.lock";
    print {$run->{LOG}} "iurt: trying to lock $lockfile\n";
    open my $lock, ">$lockfile";
    my $lock_ok;
    # lockf seems not to work, try to workarround, but this start to create lock on the lock for the lock of the file.
    my $status = 1; #File::lockf::lock($lock);
    if (!$status) {
	$lock_ok = 1;
    } else {
	print {$run->{LOG}} "ERROR iurt: could not lock pid file (status $status $!)\n";
	if (! -f "$lockfile.2") {
	    print {$run->{LOG}} "iurt: using $lockfile.2 as lock file\n";
	    open my $lock2, ">$lockfile.2" or die "FATAL iurt: could not open lock file $lockfile.2";
	    print $lock2 $$;
	    close $lock2;
	}
    }
    if (!$run->{concurrent_run}) {
	opendir my $dir, $run->{pidfile_home};
	foreach my $f (readdir $dir) {
	    my ($pid_host) = $f =~ /$pidfile\.pid\.(.*)\.pid$/ or next; 
	    if ($pid_host ne $hostname) {
		my $pf = "$run->{pidfile_home}/$f";
		open my $test_PID, $pf;
		my $pid = <$test_PID>;
		my (@stat) = stat $pf;
		my $time = $stat[9];
		my $diff = time - $time;
		my $msg = "iurt: an other iurt is running for $run->{my_arch} on $pid_host, pid $pid, since $diff seconds";
		if ($diff < 36000) {
		    print {$run->{LOG}} "$msg\n";
		    exit
		} else {
		    print {$run->{LOG}} "$msg, ignoring it\n"
		}
	    }
	}
	$run->{pidfile} .= ".pid";
    }
    $run->{pidfile} .= ".$hostname.pid";
    $pidfile = "$run->{pidfile_home}/$run->{pidfile}";
    if (-f $pidfile)  {
	my (@stat) = stat $pidfile;
	open my $test_PID, $pidfile;
	my $pid = <$test_PID>;
	close $test_PID;
	if (!$pid) {
	    print {$run->{LOG}} "ERROR iurt: invalid pidfile ($pid), should be <pid>";
	    unlink $pidfile
	}
	if ($pid && getpgrp $pid != -1) {
	    my $time = $stat[9];
	    my $state = `ps h -o state $pid`;
	    chomp $state;
	    if ($time < time - 36000 || $state eq 'Z') {
		print {$run->{LOG}} "iurt: an other iurt pid $pid is running for a very long time or is zombie, killing it\n";
		my $i;
		while ($i < 5 && getpgrp $pid != -1) {
		    kill_for_good($pid);
		    $i++;
		    sleep 1
		}
	    } else  {
		print {$run->{LOG}} "iurt: an other iurt is running for $run->{my_arch}, pid $pid, since ",time - $time," seconds\n";
		exit
	    }
	} else {
	    print {$run->{LOG}} "iurt: a previous iurt for $run->{my_arch} seems dead, cleaning.\n";
	    unlink $pidfile
	}
    }
    print {$run->{LOG}} "iurt: setting $pidfile pid lock\n";
    open my $PID, ">$pidfile" or die "FATAL iurt: could not open pidfile $pidfile for writing";
    print $PID $$;
    close $PID;
    if ($lock_ok) { 
	File::lockf::ulock($lock);
    } else {
	unlink "$lockfile.2"
    }
    close $lock;
    unlink $lockfile
}

sub check_media {
    my ($run, $cache, $config, $srpm_version, $wrong_rpm, $provides, $pack_provide, $maint) = @_;
# We could rely on only parsing the synthesis, hoping that they are correct, however this scan is very fast, so...
    foreach my $subdir (@{$config->{all_media}{$run->{media}}}) {
	my $rpms_dir = "$config->{repository}/$run->{distro}/$run->{my_arch}/media/$run->{media}/$subdir/";
	print {$run->{LOG}} "iurt: checking current packages in $rpms_dir\n";
	opendir my $rpmdir, $rpms_dir or die "Could not open $rpms_dir: $!"; 
	foreach my $rpm (readdir $rpmdir) {
	    my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm, $wrong_rpm);
	    $rarch or next;
	    $cache->{queue}{$srpm} = 1;
	    $run{status}{$srpm} = 'ok';
	    check_version($srpm, $srpm_version)
	}
	closedir $rpmdir;
    }

    foreach my $m (keys %{$config->{all_media}}) {
	foreach my $subdir (@{$config->{all_media}{$m}}) {
	    my $synthesis_file = "$config->{repository}/$run->{distro}/$run->{my_arch}/media/$m/$subdir/media_info/synthesis.hdlist.cz";
	    if (-f $synthesis_file) {
		print {$run->{LOG}} "Parsing $synthesis_file\n";
		if (open my $syn, "zcat $synthesis_file |") { 
		    my @prov;
		    my $nb;
		    while (<$syn>) {
			if (/^\@provides@(.*)/) {
			    foreach my $p (split '@', $1) {
				$p =~ /([^[]+)(?:\[(.*)\])?/g;
				push @prov, $1;
				$provides->{$1} = $2 || 1
			    }
			} elsif (/\@info\@([^@]+)@/) {
			    $nb++;
			    my $p = $1;
			    my ($name) = $p =~ /(.*)-[^-]+-[^-]+\..*$/;
			    $provides->{$p} = 1;
			    foreach (@prov) {
				$pack_provide->{$_} = $name
			    }
			    @prov = ()
			}
		    }
		    $nb < $config->{minimum_package_number} and die "FATAL iurt: synthesis files seems corrupted, only $nb packages found."
		} else {
		    die "FATAL iurt: Could not open $synthesis_file\n";
		}
	    }
	}
    }
    #"
    my $nb;
    foreach my $subdir (@{$config->{all_media}{$run->{media}}}) {
	$nb += search_packages(0, $cache, $provides, $run, $maint, $srpm_version, "$config->{repository}/$run->{distro}/SRPMS/$run->{media}/$subdir/")
    }
    $nb
}

sub search_packages {
    my ($clean, $cache, $provides, $run, $maint, $srpm_version, @dir) = @_;
    my ($to_compile, %rep, %done_rpm);
    print {$run->{LOG}} "iurt search_package: @dir\n";
    foreach my $dir (@dir) {
	print {$run->{LOG}} "iurt: checking SRPMS dir $dir\n";
	opendir my $rpmdir, $dir or next;
	foreach my $srpm (readdir $rpmdir) {
	    # this is for the output of the new svn system
	    if ($srpm =~ /^\@\d+:(.*)/) {
		link "$dir/$srpm", "$dir/$1";
		# unlink "$dir/$srpm";
		$srpm = $1
	    }
	    $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
	    $run->{status}{$srpm} ||= 0;
	    if ($config->{unwanted_packages} && $srpm =~ /$config->{unwanted_packages}/) { next }
	    my $ok = 1;
	    if (check_version($srpm, $srpm_version)) { 
		if (!$run->{ignore_failure} && defined $cache->{failure}{$srpm}) {
		    $run->{status}{$srpm} = 'build_failure';
		    next
		}
		my $check_needed = check_needed($srpm, $cache, $provides);
		$run->{status}{$srpm} = 'missing_buildrequires' if !$check_needed;
		if (!$cache->{queue}{$srpm} && $check_needed) {
		    my $hdr = RPM4::Header->new("$dir/$srpm");
		    if (!check_arch($hdr)) {
			$run->{status}{$srpm} = 'not_on_this_arch';
			next
		    }
		    my $changelog = $hdr->queryformat("%{CHANGELOGNAME}");
		    my ($mail) = $changelog =~ /<(.*@.*)>/;
		    $maint{$srpm} = $mail;
		    print "iurt: will try to compile $srpm\n";
		    $to_compile++;
		    push @{$run->{todo}}, [ $dir , $srpm, 1 ]
		}
		foreach my $arch (@{$config->{supported_arch}}) {
		    $ok &&= $cache->{queue}{$srpm}
		}
	    }
	    if ($clean && ($rep{$srpm} || $ok)) {
		print "iurt: cleaning $dir/$srpm\n";
		unlink "$dir/build/$srpm";
		unlink "$dir/$srpm"
	    }
	    $rep{$srpm} = 1
	}
	closedir $rpmdir
    }
    $to_compile
}    

sub get_date {
    my ($sec,$min,$hour,$mday,$mon,$year) = gmtime(time());
    $year += 1900;
    my $fulldate = sprintf "%4d%02d%02d%02d%02d%02d", $year, $mon+1, $mday, $hour, $min, $sec;
    my $daydate = sprintf "%4d%02d%02d", $year, $mon+1, $mday;
    $fulldate, $daydate
}

sub add_local_user {
    my ($chroot_tmp, $run, $luser, $uid) = @_;
    # change the builder user to the local user id
    # FIXME it seems that unionfs does not handle well the change of the uid of files
    # if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) {
    if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) || system(qq|sudo chroot $chroot_tmp id $luser|)) {
	print {$run->{LOG}} "ERROR: setting userid $uid to $luser in $chroot_tmp failed, trying to check the chroot\n";
	check_chroot($run->{chroot_path}, $run->{chroot_tar}, $run);
	return
    }
    if (system(qq|sudo chroot $chroot_tmp cp -R /home/builder /home/$luser|)) {
	print "ERROR iurt: could not initialized $luser directory\n";
	return
    }
    if (system(qq|sudo chroot $chroot_tmp chown -R $uid /home/$luser|)) {
	die "ERROR iurt: could not initialized $luser directory\n"; 
	return
    }
    1
}
 
sub create_temp_chroot {
    my ($run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm) = @_;
    if ($unionfs_tmp) {
	my $mount_point = "$unionfs_dir/unionfs.$run->{run}.$union_id";
	print {$run->{LOG}} "Cleaning $mount_point\n" if $run->{verbose} > 1;
	if (!clean_mnt($mount_point, $run->{verbose})) {
	    dump_cache($run);
	    die "FATAL iurt: could not kill remaining processes acceding $mount_point"
	}
	my $tmpfs;

	# we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database)
	$union_id = clean_unionfs($unionfs_dir, $run, $run->{run}, $union_id);
	$tmpfs = "$unionfs_dir/tmpfs.$run->{run}.$union_id";
	$chroot_tmp = "$unionfs_dir/unionfs.$run->{run}.$union_id";
	if (!-d $tmpfs) {
	    if (!mkdir $tmpfs) {
		print {$run->{LOG}} "ERROR iurt: Could not create $tmpfs ($!)";
		return
	    }
	}
	if (! -d $chroot_tmp) {
	    if (!mkdir $chroot_tmp) {
		print {$run->{LOG}} "ERROR iurt: Could not create $chroot_tmp ($!)";
		return
	    }
	}
	if ($cache->{no_unionfs}{$srpm}) {
	    $unionfs_tmp = 0;
	    clean_chroot($chroot_tmp, $run)
	} else {
	    # if the previous package has been built without unionfs, chroot need to be cleaned
	    clean_chroot($chroot_tmp, $run) if !$unionfs_tmp;
	    $unionfs_tmp = 1;
	    if (system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null})) {
		print {$run->{LOG}} "ERROR iurt: could not mount $tmpfs ($!)\n"; 
		return
	    }
	    if (system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run->{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|)) {
		print {$run->{LOG}} "ERROR iurt: could not mount $tmpfs and $home/chroot_$run->{distro_tag}$debug_tag with unionfs ($!)\n";
		return
	    }
	    if (system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null")) {
		print {$run->{LOG}} "ERROR iurt: could not mount /proc in the chroot $chroot_tmp ($!).\n";
		return
	    }
	    if (!-d "$chroot_tmp/dev/pts") {
		if (system("sudo mkdir $chroot_tmp/dev/pts")) {
		    print {$run->{LOG}} "ERROR iurt: could not create /dev/pts in the chroot $chroot_tmp ($!).\n";
		    return
		}
		if (system("sudo mount -t devpts none $chroot_tmp/dev/pts &>/dev/null")) {
		    print {$run->{LOG}} "ERROR iurt: could not mount /dev/pts in the chroot $chroot_tmp ($!).\n";
		    return
		}
	    }
	}
    } else {
	print {$run->{LOG}} "iurt: installing a new chroot in $chroot_tmp\n" if $run->{verbose} > 1;
	clean_chroot($chroot_tmp, $run)
    }
    $union_id, $unionfs_tmp, $chroot_tmp
}

sub add_packages {
    my ($run, $config, $chroot, $user, @packages) = @_;
    my $f;
    if (!perform_command("sudo $run{urpmi_command} @packages", 
		$run, $config,
		timeout => 300, 
		freq => 1,
		retry => 2,
		debug_mail => $run->{debug},
		error_regexp => 'cannot be installed',
		wait_regexp => {
		    'is needed by' => sub { 
		        print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; 
		        system("sudo chroot $chroot rm -rf /var/lib/rpm/__db* &> /dev/null");
		        1
		        }, 
		    'database locked' => sub { 
		        print {$run->{LOG}} "WARNING iurt: urpmi database locked, waiting...\n";
		        sleep 30; 
		        $wait_limit++; 
		        if ($wait_limit > 10) { 
		    	    $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null});
			    return
			} 
			1
		  } },)) {
	print {$run->{LOG}} "ERROR iurt: could not install @packages inside $chroot\n";
	return 0
    }
    1
}

sub add_sudoers {
    my ($run, $chroot, $user) = @_;
    my $file = "$chroot/etc/sudoers";
    my $f;
    if (!open $f, qq{| sudo sh -c "cat > $file"}) {
	print {$run->{LOG}} "ERROR iurt: could not open $file ($!)\n";
	return 0
    }
    print $f qq{Cmnd_Alias RPM=/bin/rpm,/usr/sbin/urpmi,/usr/sbin/urpme,/usr/sbin/urpmi.addmedia,/usr/sbin/urpmi.update,/usr/sbin/urpmi.removemedia
root    ALL=(ALL) ALL
$user   ALL=(ALL) NOPASSWD:RPM
};
    close $f;
    print {$run->{LOG}} "iurt: adding sudo for /bin/rpm, /usr/sbin/urpmi and /usr/sbin/urpme\n";
    -f $file or return 0;
    1
}

sub remove_chroot {
    my ($run, $dir, $func, $prefix) = @_;
    print {$run->{LOG}} "iurt remove_chroot: dir $dir all $run->{clean_all} prefix $prefix\n";
    if ($run->{clean_all}) {
	opendir my $chroot_dir, $dir;
	foreach (readdir $chroot_dir) {
	    next if !-d "$dir/$_" || /\.{1,2}/;
	    print {$run->{LOG}} "iurt: cleaning old chroot for $_ in $dir\n";
	    $func->("$dir/$_", $prefix)
	}
    } else {
	foreach my $user (@{$run->{clean}}) {
	    print {$run->{LOG}} "iurt: cleaning old chroot for $user in $dir\n";
	    $func->("$dir/$user", $prefix)
	}
    }
}

sub add_media {
    my ($run, $config, $chroot, $regexp, $media) = @_;
    print {$run->{LOG}} "iurt: adding media $run{chrooted_media} in chroot $chroot\n";
    if (!perform_command("sudo chroot $chroot urpmi.addmedia $media", 
		$run, $config,
		mail => $config->{admin},
		timeout => 300, 
		freq => 1,
		retry => 2,
		debug_mail => $run->{debug})) {
	}
    if (!check_media_added($chroot, $regexp)) { 
	print "ERROR iurt could not add media into the chroot\n";
	return
    } 
    1
}

sub check_media_added {
    my ($chroot, $media) = @_;
    my $medias = `sudo chroot $chroot urpmi.removemedia 2>&1`;
    $medias =~ /one of.* $media/m
}

sub get_local_provides {
    my ($run, $local_media) = @_;
    opendir my $dir, $local_media;
    my $urpm = new URPM;
    foreach my $d (readdir $dir) {
	$d =~ /\.rpm$/ or next;
	my $id = $urpm->parse_rpm("$local_media/$d");
	my $pkg = $urpm->{depslist}[$id];
        foreach ($pkg->provides, $pkg->files) {
	    print {$run->{LOG}} "iurt: adding $_ as provides of $d\n" if $run->{verbose} > 2;
	    $run{local_provides}{$_} = $d
	}
    }
}

sub recreate_srpm {
    my ($run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry) = @_;
# recreate a new srpm for buildarch condition in the spec file
    print {$run->{LOG}} "iurt: copying $srpm to $chroot_tmp\n" if $run->{verbose} > 1;
    perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/", 
	\%run, $config,
	mail => $config->{admin}, 
	error => "[REBUILD] cannot copy $srpm to $chroot_tmp", 
	debug_mail => $run->{debug},
	hash => "copy_$srpm") or return;

    my %opt = ( mail => $config->{admin}, 
	error => "[REBUILD] cannot install $srpm in $chroot_tmp", 
	debug_mail => $run->{debug},
	hash => "install_$srpm",
	retry => $retry,
	callback => sub { 
	    my ($opt, $output) = @_;
	    print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug};
	    if ($output =~ /warning: (group|user) .* does not exist - using root|Header V3 DSA signature/i) {
		return 1
	    } elsif ($output =~ /user $luser does not exist|cannot write to \%sourcedir/) {
		print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1;
		$opt->{error} = "[CHROOT] chroot is corrupted";
		$opt->{retry} = 1 if !$opt->{retry};
		return 
	    }
	    1
	});
    print {$run->{LOG}} "iurt: recreating src.rpm...\n" if $run->{verbose};
    if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"}, 
	    \%run, $config, %opt)) {
	print {$run->{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run->{debug};
	if ($opt{retry}) {
	    check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) or return;
	    return -1
	}
	return
    }
    # 20060515 This should not be necessairy any more if urpmi *.spec works, but it doesn't
    #
    perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm --nodeps -bs /home/$luser/rpm/SPECS/*.spec"}, 
	\%run, $config,
	mail => $config->{admin}, 
	error => "[REBUILD] cannot create $srpm in $chroot_tmp", 
	debug_mail => $run->{debug},
	hash => "create_$srpm")
}

sub get_build_requires {
    my ($run, $config, $union_id) = @_;
    $run{todo_requires} = {};
    my ($u_id, $unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or return;
    $union_id = $u_id;
    
    my $urpm = new URPM;
    foreach my $p (@{$run->{todo}}) {
	my ($dir, $srpm, $s) = @$p;
	recreate_srpm($run, $config, $chroot_tmp, $dir, $srpm, 'iurt') or return;
	$s or next;
	my $id = $urpm->parse_rpm("$dir/$srpm");
	my $pkg = $urpm->{depslist}[$id];
        foreach ($pkg->requires) {
	    print {$run->{LOG}} "iurt: adding $_ as requires of $srpm\n" if $run->{verbose} > 2;
	    $run{todo_requires}{$_} = $srpm
	}
    }
}

sub order_packages {
    my ($run, $config, $union_id, $provides, $local_media) = @_;
    my @packages = @{$run{todo}};
    my $move;
    get_local_provides($run, $local_media) or return;
    if (!$run{todo_requires}) {
	get_build_requires($run, $config, $union_id) or return
    }
    my %visit;
    my %status;
    do { 
	$move = 0;
	foreach my $p (@packages) {
	    my ($dir, $rpm, $status) = @$p;
	    defined $status{$rpm} && $status{$rpm} == 0 and next;
	    print {$run->{LOG}} "iurt: checking packages $rpm\n";
	    foreach my $r (@{$run->{todo_requires}{$rpm}}) {
		print {$run->{LOG}} "iurt: checking requires $r\n";
		if (!$run->{local_provides}{$r}) { 
		    if ($provides->{$r}) {
			$status = 1
		    } else {
			$status = 0
		    }
		} elsif ($visit{$rpm}{$r}) {
		    # to evit loops
		    $status = 0
		} elsif ($run->{done}{$rpm} && $run->{done}{$provides->{$r}}) {
		    if ($run->{done}{$rpm} < $run->{done}{$provides->{$r}}) {
			$move = 1;
			$status = $status{$provides->{$r}} + 1 
		    } else {
			$status = 0
		    }
		} elsif ($status < $status{$provides->{$r}}) {
		    $move = 1;
		    $status = $status{$provides->{$r}} + 1
		}
		$visit{$rpm}{$r} = 1;
	    }
	    $status{$rpm} = $status;
	    $p->[2] = $status
	}
    } while ($move);    
    $run->{todo} = [ sort { $a->[2] <=> $b->[2] } @packages ];
    @packages
}

sub dump_status {
    my ($local_spool, $run) = @_;
    if (open my $file, ">$local_spool/log/status.$run->{media}.log") {
	foreach my $srpm (sort keys %{$run->{status}}) {
	    print $file "$srpm: ";
	    if ($run{status}->{$srpm}) {
		print $file $run->{status}{$srpm}
	    } else {
		print $file "unknown"
	    }
	    print $file "\n"
	}
	close $file
    }
}

sub send_status_mail {
    my ($run, $config, $cache) = @_;
    print "iurt compilation status\n";
    my %output;
    foreach my $rpm (keys %{$run->{status}}) {
	next if $run->{status}{$rpm} =~ /ok|not_on_this_arch/;
	if ($run->{status}{$rpm} eq 'missing_buildrequires') {
	    foreach my $missing (keys %{$cache->{needed}{$rpm}}) {
		my $h = $cache->{needed}{$rpm}{$missing};
		my $maint = $h->{maint} || 'Other';
		my $package = $h->{package};
		if ($package) {
		    push @{$output{missing}{$maint}{$package}{$missing}{$h->{version}}}, $rpm
		} else {
		    $output{missing}{$maint}{$rpm}{$missing}{$h->{version}} = 1
		}
	    }
        } elsif ($run->{status}{$rpm} eq 'build_failure') {
	    my ($maint) = get_maint($run, $rpm);
	    if ($cache->{buildrequires}{$rpm}) {
		push @{$output{buildrequires}{$maint}}, $rpm
	    } else {
		push @{$output{build}{$maint}}, $rpm
	    }
	} elsif (!$run->{status}{$rpm}) {
	    # need to find something more usefull to do at that point
	    next
	}
    }
    my $text = "*** Missing buildrequires tag in specfile ***\n";
    foreach my $maint (keys %{$output{buildrequires}}) {
	$text .= "\n$maint\n";
	foreach my $pack (keys %{$output{missing}{$maint}}) {
	    foreach my $missing (keys %{$cache->{buildrequires}{$pack}}) {
		my $rpms = $cache->{buildrequires}{$pack}{$missing};
		if (@$rpms) {
		    $text .= "  $pack should have a buildrequires on @$rpms (for $missing-devel)\n"
		} else {
		    $text .= "  $pack should have a buildrequires for $missing-devel\n"
		}
	    }
	}
    }

    my $text = "*** Missing dependencies ***\n";
    foreach my $maint (keys %{$output{missing}}) {
	$text .= "\n$maint\n";
	foreach my $pack (keys %{$output{missing}{$maint}}) {
	    foreach my $missing (%{$output{missing}{$maint}{$pack}}) {
		my $h = $output{missing}{$maint}{$pack}{$missing};
		foreach my $version (keys %$h) {
		    if (ref $h->{$version}) {
			$text .= "  $pack should be recompile because\n  $missing ". ($version ? "$version " : ''). "is not provided anymore\n";
			$text .= "    to compile ". join("\n               ", @{$h->{$version}}). "\n"
		    } else {
			$text .= "  $pack needs $missing ". ($version ? "$version " : ''). "\n";
		    }
		}
	    }
	}
    }
    $text .=  "\n*** Build failure ***\n";
    foreach my $maint (keys %{$output{build}}) {
	$text .= "\n$maint\n";
	foreach my $rpm (@{$output{build}{$maint}}) {
	    $text .= "  $rpm (see $config->{log_url}/$run{distro_tag}/$run{my_arch}/$run->{media}/log/$rpm/)\n"
	}
    }
    print "$text\n";
    sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", 0, 0, 0)
}

sub get_maint {
    my ($run, $srpm) = @_;
    my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.[^.]+$/;
    $srpm_name ||= $srpm;
    if ($run{maint}{$srpm}) {
	return $run{maint}{$srpm}, $srpm_name
    }
    my $maint = `rpmmon -s -p "$srpm_name"`;
    chomp $maint;
    $run{maint}{$srpm} = $maint;
    $maint, $srpm_name
}

sub find_provides {
    my ($run, $pack_provide, $p) = @_;
    my @rpm;
    foreach my $provides (keys %{pack_provide}) {
	if ($provides =~ /$p/ && $provides =~ /devel/) {
	    push @rpm, $pack_provide->{$provides};
	}
    }
    @rpm
}