iurt2

1 files changed, 58 insertions, 42 deletions

diff --git a/iurt2 b/iurt2
index 07530e2..727ff6f 100755
--- a/iurt2
+++ b/iurt2
@@ -50,6 +50,9 @@ $run{todo} = [ ];
     [ "c", "chroot", 0, "", 
     "Check chroot and update it if needed",
     sub { $run{chroot} = 1 }, "Activating debug mode" ],
+    [ "", "config", 2, "<configuration keyword> <value>", 
+    "Override a configuration file variable",
+    sub { my ($key, $value) = @_; $run{config}{$key} = $value }, "Overriding configuration variable" ],
     [ "", "config_help", 0, "", 
     "Explain configuration files keywords", 
     sub { $run{config_usage} = 1 }, "Activating debug mode" ],
@@ -64,11 +67,12 @@ $run{todo} = [ ];
     sub { 
 	$run{log} = pop @_; 
 	open $run{LOG}, ">$run{log}" or die "unable to open $run{log}\n";
-	print *{$run{LOG}}, "command line: @ARGV\n"
+	print *{$run{LOG}}, "command line: @ARGV\n";
+	1
     }, "Log file" ],
    [ "m", "media", -1, "", 
     "Media to rebuild", 
-    sub { ($run{media}) = @_ }, "Adding a media to rebuild" ],
+    sub { ($run{media}) = @_; 1 }, "Adding a media to rebuild" ],
     [ "r", "rebuild", -2, "<distro> <architecture> <srpm 1> <srpm 2> ... <srpm n>", 
     "Rebuild the packages, e.g. $program_name -r cooker x86_64 /home/foo/rpm/SRPMS/foo-2.3-12mdk.src.rpm", 
     sub { 
@@ -83,7 +87,8 @@ $run{todo} = [ ];
 	    } else {
 		die "FATAL iurt: $_ does not seems to be a SRPM\n"
 	    }
-	}    
+	}
+	1
 	}, "Activating rebuild mode" ],
     [ "", "upload", [
 	["", "upload", 0, "[options]", 
@@ -98,15 +103,18 @@ $run{todo} = [ ];
     ], "[options]", 
     "Upload the rebuild packages", 
     sub { $run{upload} = 1 }, "Setting the upload flag" ],
+    [ "", "no_rsync", 0, "", 
+    "Do not send build log to the distant rsync server", 
+    sub { $run{no_rsync} = 1 }, "Setting the no rsync warn flag" ],
     [ "w", "warn", 0, "", 
     "Warn maintainer of the packages about problem in the rebuild", 
-    sub { $run{warn} = @_ }, "Setting warn flag to warn maintainers" ],
+    sub { $run{warn} = @_; 1 }, "Setting warn flag to warn maintainers" ],
     [ "d", "dir", -1, "", 
     "Directory where to find packages to rebuild", 
-    sub { $run{extra_dir} = @_ }, "Adding extra source packages directories" ],
+    sub { $run{extra_dir} = @_; 1 }, "Adding extra source packages directories" ],
 );
 
-open(my $LOG, ">&$run{LOG}");	
+open(my $LOG, ">&STDERR");
 $run{LOG} = $LOG;
 
 my $todo = parseCommandLine("$program_name", \@ARGV, \@params);
@@ -116,8 +124,6 @@ foreach my $t (@$todo)  {
     &{$t->[0]}(@{$t->[1]}) or print {$run{LOG}} "ERROR: $t->[2]\n";
 }
 
-# maybe time to switch to $program_name::parse_command_line
-
 $run{distro_tag} = $run{distro};
 $run{distro_tag} =~ s,/,-,g;
 my $urpmi_options = "-v --no-verify-rpm --nolock --auto";
@@ -151,6 +157,7 @@ my %config_usage = (
     home => { desc => 'Home dir', default => $HOME },
     install_chroot_binary => { desc => 'Tool used to create initial chroot', default => 'install-chroot-tar.sh' },
     local_home => { desc => 'Where to build packages', default => $HOME },
+    local_upload => { desc => 'Where to store build packages and log', default => '' },
     log_size_limit => { desc => 'Maximum authorized size for a log file', default => '100M' },
     log_size_date => { desc => 'Number of days log should be kept', default => '30' },
     minimum_package_number => { "Minimum number of packages in a synthesis file to consider it valid", default => 1000 },
@@ -169,7 +176,7 @@ if (!$arch_comp{$real_arch}{$run{my_arch}}) {
     die "FATAL iurt: could not compile $run{my_arch} binaries on a $real_arch"
 }
 foreach my $k (keys %config_usage) {
-    $config->{$k} ||= $config_usage{$k}{default}
+    $config->{$k} ||= $run{config}{$k} || $config_usage{$k}{default}
 }
 
 $config->{upload} .= $run{distro};
@@ -189,7 +196,6 @@ if (!$run{debug} && $run{media} || $run{chroot}) {
     check_pid(\%run)
 }
 
-
 my $debug_tag = '_debug' if $run{debug};
 my $chroot = "$config->{local_home}/chroot$debug_tag";
 my $chroot_tar = "$chroot-$run{distro_tag}.$run{my_arch}.tar.gz";
@@ -199,41 +205,48 @@ if ($run{chroot}) {
 
 my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.cache";
 my $cache;
-if (-f $cachefile || $run{cache}) {
+if (-f $cachefile && $run{cache}) {
     print {$run{LOG}} "iurt: loading cache file $cachefile\n";
     $cache = do $cachefile
 } else {
-    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {}, no_unionfs => {} }
+    $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} }
 }
 
 my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint);
+$to_compile = @{$run{todo}};
 $to_compile += check_media(\%run, $cache, $config, \%srpm_version, \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media};
 $to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{extra};
 
 dump_cache();
 
 if (!@{$run{todo}} && !$run{debug}) {
-    print "iurt: nothing to do\n";
+    print {$run{LOG}} "iurt: nothing to do\n";
     unlink $run{pidfile} if $run{pidfile};
     exit
 }
-print "iurt: will try to compile $to_compile packages\n";
+print {$run{LOG}} "iurt: will try to compile $to_compile packages\n";
 
 exit if !$run{rebuild};
 
-$cache->{run} ||= 1;
-$run{run} = ++$cache->{run};
+if ($run{cache}) {
+    $run{run} = ++$cache->{run} 
+} else {
+    my ($sec,$min,$hour,$mday,$mon,$year) = gmtime(time());
+    $year += 1900;
+    my $fulldate = sprintf "%4d%02d%02d%02d%02d%02d", $year, $mon+1, $mday, $hour, $min, $sec;
+    $run{run} = "0.$fulldate"
+}
 
-my $local_spool = "$config->{local_home}/iurt/$run{distro_tag}/$run{my_arch}";
-if (!-d "$config->{local_home}/iurt/$run{distro_tag}/") {
-    mkdir "$config->{local_home}/iurt/$run{distro_tag}";
+$config->{local_upload} ||= $config->{local_home};
+my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}";
+if (!-d "$config->{local_upload}/iurt/$run{distro_tag}/") {
+    mkdir "$config->{local_upload}/iurt/$run{distro_tag}";
     if (!-d $local_spool) {
 	mkdir $local_spool;
 	mkdir "$local_spool/log"
     }
 }
 
-
 dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros";
 my $unionfs_dir;
 if ($run{unionfs}) {
@@ -243,12 +256,12 @@ if ($run{unionfs}) {
 }
 
 my $s = sub { 
-    print "iurt: dumping cache...\n"; 
+    print {$run{LOG}} "iurt: dumping cache...\n"; 
     dump_cache();
     $Data::Dumper::Indent = 0;
     $Data::Dumper::Terse = 1;
-    print "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n"; 
-    print "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n";
+    print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n"; 
+    print {$run{LOG}} "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n";
     exit 
 };
 $SIG{TERM} = $s;
@@ -267,8 +280,9 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 	check_version($srpm) or next;
 	if ($run{debug}) { $run{debug}++ == 2 and exit }
 	$done++;
-	print "iurt: packages $srpm [$done/$to_compile]\n";
+	print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n";
 	# FIXME unfortunately urpmi stalls quite often
+retry:
 	my $match = "urpmi $urpmi_options --root $chroot";
 	if (!clean_process($match)) {
 	    dump_cache();
@@ -318,7 +332,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 	#($maintainer, $cc) = ($config->{admin},'');
 	
 	# recreate a new srpm for buildarch condition in the spec file
-	print "Copying $srpm to $chroot\n";
+	print {$run{LOG}} "Copying $srpm to $chroot\n";
 	perform_command("sudo cp $dir/$srpm $chroot/home/builder/rpm/SRPMS/", 
 	        \%run, $config,
 		mail => $config->{admin}, 
@@ -337,16 +351,16 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 		error => "[REBUILD] cannot create $srpm in $chroot", 
 		debug_mail => $run{debug},
 		hash => "create_$srpm") or next;
-	print "Installing build dependencies of $srpm...\n";
+	print {$run{LOG}} "Installing build dependencies of $srpm...\n";
 	
 	perform_command("sudo urpmi $urpmi_options --root $chroot $chroot/home/builder/rpm/SRPMS/$srpm", 
 		\%run, $config,
-		mail => $maintainer, 
+		mail => $config->{admin},
 		error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}",  
 		hash => "install_deps_$srpm", 
 		timeout => 600, 
 		freq => 1,
-		cc => $cc, 
+		#cc => $cc, 
 		debug_mail => $run{debug},
 		error_regexp => 'cannot be installed',
 		wait_regexp => 'database locked',
@@ -355,14 +369,14 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 		callback => sub { 
 			my ($opt, $output) = @_;
 			print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
-			my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)| \[(.*)\])?\)/g;
+			my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
 			@missing_deps or return;
 			while (my $missing_deps = shift @missing_deps) {
 			    my $version = shift @missing_deps;
 			    my $version2 = shift @missing_deps;
 			    $version ||= $version2 || 0;
 			    my $p = $pack_provide{$missing_deps} || $missing_deps;
-			    my $other_maint = `rpmmon -p $p`;
+			    my $other_maint = `rpmmon -p "$p"`;
 			    chomp $other_maint;
 			    print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
 			    if ($other_maint && $other_maint ne 'NOT_FOUND') {
@@ -377,7 +391,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 	# try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory
 	# running rpm -qa several time seems to fix the problem
 	foreach (1 .. 3) {
-	    system("sudo chroot $chroot rpm -qa")
+	    system("sudo chroot $chroot rpm -qa &> /dev/null")
 	}
 	perform_command("sudo chroot $chroot rpm -qa", 
 		\%run, $config,
@@ -385,7 +399,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 		timeout => 60, 
 		debug_mail => $run{debug},
 		log => "$local_spool/log/"); # or next; As this failed quite often, do not stop
-	print "Compiling $srpm\n";
+	print {$run{LOG}} "Compiling $srpm\n";
 	if (!perform_command(qq{TMP=/home/builder/tmp/ sudo chroot $chroot /bin/su builder -c "rpm --rebuild /home/builder/rpm/SRPMS/$srpm"}, 
 			\%run, $config,
 			mail => $maintainer, 
@@ -424,6 +438,8 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 			$cache->{failure}{$srpm} = 1
 		    } elsif (!$cache->{no_unionfs}{$srpm}) {
 			$cache->{failure}{$srpm} = 1
+		    } else {
+			goto retry
 		    }
 		next
 	}
@@ -443,10 +459,10 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
 		next
 	}
 	if ($run{debug}) {
-	    print "iurt: debug mode, skip other packages\n";
+	    print {$run{LOG}} "iurt: debug mode, skip other packages\n";
 	    exit
 	} else {
-	    system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_spool") and print "ERROR: could not copy rpm files from $chroot/home/builder/rpm/RPMS/ to $local_spool ($!)\n";
+	    system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_spool") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot/home/builder/rpm/RPMS/ to $local_spool ($!)\n";
 	    process_queue($config, \%run, \@wrong_rpm)
 	}
 }
@@ -463,9 +479,9 @@ if (open my $file, ">$local_spool/log/wrong_srpm_names.log") {
     }
 }
 
-if ($config->{rsync_to}) {
+if ($config->{rsync_to} && !$run{no_rsync}) {
     	# remove some old and very big log files not to saturate the server
-	system(qq{find $local_spool/log/ -name "*.log" \( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \) -exec rm -f {} \;});
+	system(qq|find $local_spool/log/ -name "*.log" \( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \) -exec rm -f {} \;|);
 	system("rsync --delete -alHPe 'ssh -c arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/");
 }
 
@@ -548,8 +564,8 @@ sub clean {
     while ($ps = `$cmd "$var"`) {
 	system(qq{$kill_cmd "$var"});
 	sleep 1;
-	chomp $ps;
-	print {$run{LOG}} "Trying to removed previous blocked processes for $var ($ps)\n";
+	$ps =~ s/\n/,/g;
+	print {$run{LOG}} "Trying to remove previous blocked processes for $var ($ps)\n";
 	return 0 if $i++ > 10
     }
     1
@@ -621,7 +637,7 @@ sub check_needed {
 sub process_queue {
     my ($config, $run, $wrong_rpm) = @_;
     return if !$run->{upload};
-    my $dir = "$config->{local_home}/iurt/$run->{distro_tag}/$run->{my_arch}";
+    my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}";
     opendir my $rpmdir, $dir or next;
     foreach my $rpm (readdir $rpmdir) {
 	my ($rarch, $srpm) = update_srpm($dir, $rpm, $wrong_rpm);
@@ -753,14 +769,14 @@ sub perform_command {
 	    $output = <$log>
 	}
 	my $err = $?;
+	if (ref $opt{callback}) {
+	    $opt{callback}(\%opt, $output)
+	}
 	if ($kill) {
 	    $output = "Command has been killed after $opt{timeout} seconds: $command\n$output"
 	} else {
 	    $output = "Command failed: $command\n$output"
 	}
-	if (ref $opt{callback}) {
-	    $opt{callback}(\%opt, $output)
-	}
 	if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/) {
 	    $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback};
 	    print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n";