- add concurrent-run option to run several instance and merge cache

- add support of urpmi --use-distrib to compile on other version than the system one
- add support of function in configuration for more complex default values
- create a pidfile including other hosts too
- improve dump_cache to support concurrent run (not really tested yet)
- fork to monitor log file size and limit it

1 files changed, 135 insertions, 42 deletions

diff --git a/iurt2 b/iurt2
index 6753093..e701a55 100755
--- a/iurt2
+++ b/iurt2
@@ -47,10 +47,10 @@ $run{todo} = [ ];
     [ "", "$program_name", 0, "[options]", 
     "$program_name rebuild bot", 
     sub { $arg or usage($program_name, \@params) }, "" ],
-    [ "d", "distro", 1, "", 
+    [ "", "distro", 1, "<distro>", 
     "Set the distribution",
     sub { ($run{distro}) = @_; 1 }, "Setting the distribution" ],
-    [ "a", "arch", 1, "", 
+    [ "a", "arch", 1, "<architecture>", 
     "Set the architecture",
     sub { ($run{my_arch}) = @_; 1 }, "Setting architecture" ],
     [ "", "cache", 0, "", 
@@ -62,6 +62,12 @@ $run{todo} = [ ];
     [ "c", "chroot", 0, "", 
     "Check chroot and update it if needed",
     sub { $run{chroot} = 1 }, "Activating chroot updating" ],
+    [ "", "concurrent-run", 0, "", 
+    "Allow several iurt to run on different machines (slower)",
+    sub { $run{concurrent_run} = 1 }, "Activating concurrent run checks" ],
+    [ "d", "dir", -1, "", 
+    "Directory where to find packages to rebuild", 
+    sub { $run{extra_dir} = \@_; 1 }, "Adding extra source packages directories" ],
     [ "", "config", 2, "<configuration keyword> <value>", 
     "Override a configuration file variable",
     sub { my ($key, $value) = @_; $run{config}{$key} = $value }, "Overriding configuration variable" ],
@@ -126,6 +132,9 @@ $run{todo} = [ ];
     [ "", "no_rsync", 0, "", 
     "Do not send build log to the distant rsync server", 
     sub { $run{no_rsync} = 1 }, "Setting the no rsync warn flag" ],
+    [ "", "use-system-distrib", 1, "<media>", 
+    "Use the current system urpmi configuration", 
+    sub { $run{use_system_distrib} = shift; 1 }, "Setting system distrib for urpmi configuration" ],
     [ "v", "verbose", 1, "<verbose level>", 
     "Give more info messages about what is going on (level from 1 to 10)", 
     sub { $run{verbose} = $_[0]; 1 }, "Setting verbose level" ],
@@ -133,11 +142,8 @@ $run{todo} = [ ];
     "Warn maintainer of the packages about problem in the rebuild", 
     sub { $run{warn} = 1 ; 1 }, "Setting warn flag to warn maintainers" ],
     [ "", "stop", 1, "<rpm step>", 
-    "Perform rpm -b<rpm step> (p c i) instead of rpm -ba and then open a shell in the chroot", 
+    "Perform rpm -b<rpm step> (p c i l b a s) instead of rpm -ba and then open a shell in the chroot", 
     sub { ($run{stop}) = @_; 1 }, "Setting rpm build option" ],
-    [ "d", "dir", -1, "", 
-    "Directory where to find packages to rebuild", 
-    sub { $run{extra_dir} = \@_; 1 }, "Adding extra source packages directories" ],
 );
 
 open(my $LOG, ">&STDERR");
@@ -152,7 +158,6 @@ foreach my $t (@$todo)  {
 
 $run{distro_tag} = $run{distro};
 $run{distro_tag} =~ s,/,-,g;
-my $urpmi_options = "-v --no-verify-rpm --nolock --auto";
 
 my $real_arch = `uname -m`;
 chomp $real_arch;
@@ -173,11 +178,17 @@ if (-f $configfile) {
 } else {
     $config = {}
 }
-
+my $urpmi_options = "-v --no-verify-rpm --nolock --auto";
+if ($run{use_system_distrib}) {
+    $config->{basesystem_media_root} ||= $run{use_system_distrib}
+} else {
+    $urpmi_options .= " --use-distrib $config->{repository}/$run{distro}/$run{my_arch}"
+}
 my %config_usage = ( 
     admin => { desc => 'Mail of the administrator of packages builds', default => '' },
     all_media => { desc => 'List of known media', default => [ 'main', 'contrib' ] },
-    basesystem_media => { desc => 'Where to find basesystem packages', default => "$config->{repository}/$run{distro}/$run{my_arch}/media/main/" },
+    basesystem_media_root => { desc => 'Name of the media holding basesystem packages', default => sub { my ($config, $run) = @_; "$config->{repository}/$run{distro}/$run{my_arch}/" } },
+    basesystem_media => { desc => 'Where to find basesystem packages', default => 'main' },
     cache_home => { desc => 'Where to store the cache files', default => "$HOME/.bugs" },
     distribution => { desc => 'Name of the packages distribution', default => 'Mandriva Linux' },
     home => { desc => 'Home dir', default => $HOME },
@@ -202,8 +213,14 @@ if (!$arch_comp{$real_arch}{$run{my_arch}}) {
     die "FATAL iurt: could not compile $run{my_arch} binaries on a $real_arch"
 }
 foreach my $k (keys %config_usage) {
+    ref $config_usage{$k}{default} eq 'CODE' and next;
     $config->{$k} ||= $run{config}{$k} || $config_usage{$k}{default}
 }
+foreach my $k (keys %config_usage) {
+    ref $config_usage{$k}{default} eq 'CODE' or next;
+    my $a = $config_usage{$k}{default}($config, \%run);
+    $config->{$k} ||= $run{config}{$k} || $a
+}
 
 $config->{upload} .= $run{distro};
 $config->{upload} =~ s/community//g;
@@ -218,18 +235,20 @@ if ($run{distro} ne 'cooker') {
 -d $config->{upload} or usage($program_name, \@params, "$config->{upload} does not exist");
 
 if (!$run{debug} && $run{media} || $run{chroot}) {
-    $run{pidfile} = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.pid";
+    $run{pidfile_home} = "$config->{cache_home}/";
+    $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}";
     check_pid(\%run)
 }
 
 my $debug_tag = '_debug' if $run{debug};
-my $chroot = "$config->{local_home}/chroot$debug_tag";
-my $chroot_tar = "$chroot-$run{distro_tag}.$run{my_arch}.tar.gz";
+my $chroot = "$config->{local_home}/chroot_$run{distro_tag}$debug_tag";
+my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
 if ($run{chroot}) {
     check_chroot($chroot, $chroot_tar, \%run)
 }
 
 my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.cache";
+$run{cachefile} = $cachefile;
 my $cache;
 if (-f $cachefile && $run{cache}) {
     print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
@@ -237,17 +256,18 @@ if (-f $cachefile && $run{cache}) {
 } else {
     $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
 }
+$run{cache} = $cache;
 
 my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint);
 $to_compile = @{$run{todo}};
 $to_compile += check_media(\%run, $cache, $config, \%srpm_version, \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media};
 $to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{extra};
 
-dump_cache();
+dump_cache(\%run);
 
 if (!@{$run{todo}} && !$run{debug}) {
     print {$run{LOG}} "iurt: nothing to do\n";
-    unlink $run{pidfile} if $run{pidfile};
+    unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
     exit
 }
 print {$run{LOG}} "iurt: will try to compile $to_compile packages\n" if $run{verbose} > 1;
@@ -273,8 +293,10 @@ if (!-d "$config->{local_upload}/iurt/$run{distro_tag}/") {
 	mkdir "$local_spool/log"
     }
 }
+# perform some cleaning before running to have some more space
+system(qq|find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;|);
 
-dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros";
+dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros";
 my $unionfs_dir;
 if ($run{unionfs}) {
     # FIXME need to grep /proc/modules not ot try to load it if already loaded
@@ -286,7 +308,7 @@ if ($run{unionfs}) {
 
 my $s = sub { 
     print {$run{LOG}} "iurt: dumping cache...\n"; 
-    dump_cache();
+    dump_cache(\%run);
     $Data::Dumper::Indent = 0;
     $Data::Dumper::Terse = 1;
     print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n"; 
@@ -297,6 +319,7 @@ $SIG{TERM} = $s;
 $SIG{INT} = $s;
 
 my %done;
+$run{done} = \%done;
 my $wait_limit;
 my $done;
 my $home = $config->{local_home};
@@ -314,14 +337,14 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 retry:
 	my $match = "urpmi $urpmi_options --root $chroot";
 	if (!clean_process($match, $run{verbose})) {
-	    dump_cache();
+	    dump_cache(\%run);
 	    die "FATAL iurt: Could not have urpmi working !"
 	}
 	if ($unionfs_tmp) {
 	    my $mount_point = "$unionfs_dir/unionfs.$run{run}.$union_id";
 	    print {$run{LOG}} "Cleaning $mount_point\n" if $run{verbose} > 1;
 	    if (!clean_mnt($mount_point, $run{verbose})) {
-		dump_cache();
+		dump_cache(\%run);
 		die "FATAL iurt: could not kill remaining processes acceding $mount_point"
 	    }
 	    my $tmpfs;
@@ -340,7 +363,7 @@ retry:
 		clean_chroot($chroot, \%run) if !$unionfs_tmp;
 		$unionfs_tmp = 1;
 		system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null}) and die "FATAL iurt: could not mount $tmpfs ($!)";
-		system(qq{sudo mount -o dirs=$tmpfs=rw:$home/chroot$debug_tag=ro -t unionfs none $chroot &>/dev/null}) and die "FATAL iurt: could not mount $tmpfs and $home/chroot with unionfs ($!)";
+		system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run{distro_tag}$debug_tag=ro -t unionfs none $chroot &>/dev/null|) and die "FATAL iurt: could not mount $tmpfs and $home/chroot_$run{distro_tag}$debug_tag with unionfs ($!)";
 		system("sudo mount -t proc none $chroot/proc &>/dev/null") and die "FATAL iurt: could not mount /proc in the chroot $chroot.";
 	    }
 	} else {
@@ -505,12 +528,14 @@ retry:
 	    }
 	    process_queue($config, \%run, \@wrong_rpm, 1)
 	}
+	# dymp_cache each time so that concurrent process can get updated
+	dump_cache(\%run) if $run{concurrent_run}
 }
 
 print {$run{LOG}} "iurt: reprocess generated packages queue\n" if $run{verbose};
 process_queue($config, \%run, \@wrong_rpm);
 
-dump_cache();
+dump_cache(\%run);
 
 print {$run{LOG}} "ERROR iurt: RPM with a wrong SRPM name\n" if @wrong_rpm;
 if (open my $file, ">$local_spool/log/wrong_srpm_names.log") {
@@ -530,7 +555,7 @@ print {$run{LOG}} "iurt: try to clean remaining unionfs\n" if $run{verbose};
 if ($run{unionfs}) {
     clean_all_unionfs($unionfs_dir)
 }
-unlink $run{pidfile} if $run{pidfile};
+unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
 exit;
 
 sub config_usage {
@@ -717,20 +742,36 @@ sub update_srpm {
 }
 
 sub dump_cache {
-	my $filename = $cachefile;
-	# Right now there are no mechanism of concurrent access/write to the cache. There is 
-	# on global lock for one iurt session. A finer cache access would allow several iurt running
-	# but the idea is more to have a global parrallel build than several local ones.
-	return if $run{debug} || !$run{cache};
-	open my $file, ">$filename.tmp" or die "FATAL iurt dump_cache: cannot open $filename.tmp";
-	flock($file,LOCK_EX);
-	seek($file, 0, 2);
-	$Data::Dumper::Indent = 1;
-	$Data::Dumper::Terse = 1;
-	print $file Data::Dumper->Dump([ $cache ], [ "cache" ]);
-	unlink $filename;
-	link "$filename.tmp", $filename;
-	flock($file,LOCK_UN)
+    my ($run) = @_;
+    my $filename = $run->{cachefile};
+    my $cache = $run->{cache};
+    # Right now there are no mechanism of concurrent access/write to the cache. There is 
+    # on global lock for one iurt session. A finer cache access would allow several iurt running
+    # but the idea is more to have a global parrallel build than several local ones.
+    return if $run->{debug} || !$run->{cache};
+    open my $file, ">$filename.tmp" or die "FATAL iurt dump_cache: cannot open $filename.tmp";
+    flock($file,LOCK_EX);
+    seek($file, 0, 2);
+    if ($run{concurrent_run}) {
+	print {$run{log}} "iurt: merging cache";
+	my $old_cache;
+	if (-f $filename) {
+	    print {$run{LOG}} "iurt: loading cache file $cachefile\n" if $run{verbose} > 1;
+	    $old_cache = do $cachefile;
+	    foreach my $k ('rpm_srpm', 'failure', 'no_unionfs', 'queue', 'needed', 'warning') {
+		foreach my $rpm (%{$old_cache->{$k}}) {
+		    $cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm}
+		}
+	    }
+	}
+	#  $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
+    }
+    $Data::Dumper::Indent = 1;
+    $Data::Dumper::Terse = 1;
+    print $file Data::Dumper->Dump([ $cache ], [ "cache" ]);
+    unlink $filename;
+    link "$filename.tmp", $filename;
+    flock($file,LOCK_UN)
 }
 
 sub sendmail {
@@ -791,25 +832,51 @@ sub perform_command {
 	print "Would have rum $command with a timeout of $opt{timeout}\n";
 	return 1
     }
+    my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log";
+    my $pid;
+    if ($opt{log}) {
+	my $parent_pid = $$;
+	$pid = fork;
+	my $tot_time;
+	if (!$pid) {
+	    local $SIG{ALRM} = sub { exit };
+	    $tot_time += sleep 60;
+	    my $size_limit = $config->{log_size_limit};
+	    $size_limit =~ s/k/000/i;
+	    $size_limit =~ s/M/000000/i;
+	    $size_limit =~ s/G/000000000/i;
+	    while ($tot_time < $opt{timeout}) {
+		$tot_time += sleep 60;
+		my (@stat) = stat $logfile;
+		if ($stat[7] > $size_limit) {
+		    print {$run{LOG}} "WARNING: killing current command because of log size exceeding limit ($size_limit > $config->{log_size_limit})\n";
+		    kill 14, $parent_pid;
+		    exit
+		}
+	    }
+	    exit
+	}
+    }
     eval {
 	local $SIG{ALRM} = sub { print "Timeout!\n"; $kill = 1; die "alarm\n" }; # NB: \n required
 	alarm $opt{timeout};
 	print {$run{LOG}} "$command\n" if $run{verbose} > 2;
 	if ($opt{log}) {
 	    #$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
-	    system("$command &> $opt{log}/$opt{hash}.$run->{run}.log");
+	    system("$command &> $logfile");
 	} else {
 	    $output = `$command 2>&1`;
 	}
 	alarm 0;
     };
+    kill 14, $pid if $pid;
     if ($@) {
 	# timed out
 	die unless $@ eq "alarm\n";   # propagate unexpected errors
 	return 0
     } else {
 	# Keep the run first on the harddrive so that one can check the command status tailing it
-	if ($opt{log} && open my $log, "$opt{log}/$opt{hash}.$run->{run}.log") {
+	if ($opt{log} && open my $log, $logfile) {
 	    local $/;
 	    $output = <$log>
 	}
@@ -827,7 +894,7 @@ sub perform_command {
 	    print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n";
 	    sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$opt{wait_regexp}\n\n$output", 0, 0, $opt{debug_mail});
 	    if ($opt{die}) {
-		dump_cache();
+		dump_cache($run);
 		die "FATAL iurt: $opt{error}."
 	    }
 	    return 0
@@ -843,7 +910,7 @@ sub perform_command {
 	    $cache->{warning}{$opt{hash}}{$opt{mail}}++;
 	    print {$run->{LOG}} "\n$output\n";
 	    if ($opt{die}) {
-		dump_cache();
+		dump_cache($run);
 		die "FATAL iurt: $opt{error}."
 	    }
 	    return 0
@@ -870,7 +937,7 @@ sub check_chroot {
     print {$run{LOG}} "iurt: checking basesystem tar\n" if $run{verbose};
     system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot" &> /dev/null});
     clean_chroot($chroot, $run, 1) or die "FATAL iurt: Could no prepare initial chroot";
-    perform_command("sudo $config->{install_chroot_binary} cooker $config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build", 
+    perform_command("sudo $config->{install_chroot_binary} $config->{basesystem_media_root} $config->{basesystem_media_root}/media/$config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build", 
 	$run, $config,
 	mail => $config->{admin},
 	error => "[REBUILD] Creating the inital chroot for $run->{distro_tag} on $run->{my_arch} failed", 
@@ -887,19 +954,44 @@ sub dump_rpmmacros {
 \%_tmppath               \%(echo \$HOME)/rpm/tmp/
 \%distribution           $config->{distribution}
 \%vendor                 $config->{vendor}
-\%packager               $config->{packager}}
+\%packager               $config->{packager}};
+	-f $file or return 0
 }
 
 sub check_pid {
     my ($run) = @_;
+    my $hostname = `hostname`;
+    chomp $hostname;
     my $pidfile = $run->{pidfile};
-    open my $lock, "$pidfile.lock";
+    open my $lock, "$run->{pidfile_home}/$pidfile.$hostname.pid.lock";
     flock($lock,LOCK_EX);
+    if (!$run->{concurrent_run}) {
+	opendir my $dir, $run->{pidfile_home};
+	foreach my $f (readdir $dir) {
+	    my ($pid_host) = $f =~ /$pidfile\.pid\.(.*)\.pid$/ or next; 
+	    if ($pid_host ne $hostname) {
+		my $pf = "$run->{pidfile_home}/$f";
+		open my $test_PID, $pf;
+		my $pid = <$test_PID>;
+		my (@stat) = stat $pf;
+		my $time = $stat[9];
+		print {$run->{LOG}} "iurt: an other iurt is running for $run->{my_arch} on $pid_host, pid $pid, since ",time - $time," seconds\n";
+		exit
+	    }
+	}
+	$run->{pidfile} .= ".pid";
+    }
+    $run->{pidfile} .= ".$hostname.pid";
+    $pidfile = "$run->{pidfile_home}/$run->{pidfile}";
     if (-f $pidfile)  {
 	my (@stat) = stat $pidfile;
 	open my $test_PID, $pidfile;
 	my $pid = <$test_PID>;
 	close $test_PID;
+	if (!$pid) {
+	    print {$run->{LOG}} "ERROR iurt: invalid pidfile ($pid), should be <pid>";
+	    unlink $pidfile
+	}
 	if ($pid && getpgrp $pid != -1) {
 	    my $time = $stat[9];
 	    if ($time < time - 36000) {
@@ -919,6 +1011,7 @@ sub check_pid {
 	    unlink $pidfile
 	}
     }
+    print "iurt: setting $pidfile pid lock\n";
     open my $PID, ">$pidfile" or die "FATAL iurt: could not open pidfile $pidfile for writing";
     print $PID $$;
     close $PID;