- start to use a wrapper for sudo commands

- add suboptions to dkms to limit media and kernels
- use a special cache file for dkms mode
- use the new check_build_chroot functions
- use urpmi->install_packages systematically
- do not add macros on the initial chroot, it will be done later
- move check_chroot functions to Iurt::Chroot

1 files changed, 64 insertions, 69 deletions

diff --git a/iurt2 b/iurt2
index 0e73796..b8acbf1 100755
--- a/iurt2
+++ b/iurt2
@@ -36,7 +36,7 @@ use Data::Dumper;
 use URPM;
 use Iurt::DKMS;
 use Iurt::Urpmi;
-use Iurt::Chroot qw(clean_chroot dump_rpmmacros add_local_user create_temp_chroot remove_chroot clean_all_chroot_tmp clean_unionfs clean_chroot_tmp);
+use Iurt::Chroot qw(clean_chroot add_local_user create_temp_chroot remove_chroot clean_all_chroot_tmp clean_unionfs clean_chroot_tmp check_build_chroot);
 use Iurt::Process qw(perform_command clean kill_for_good);
 use Iurt::Mail qw(sendmail);
 use File::NCopy qw(copy);
@@ -69,7 +69,7 @@ $run{todo} = [ ];
             [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}]
  	    [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs]
  	    [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename]  [--unionfs] [--status] [--ignore-failure]
- 	    {--config_help | --dkms 
+ 	    {--config_help | --dkms {--media <media regexp>}
 	    --chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } |
  	    --rebuild {cooker|2006.0|community/2006.0|...} {i586|x86_64|ppc|...} {filename1.src.rpm} {filename2.src.rpm} ... {filenamen.src.rpm} }", 
     "$program_name is a perl script to rebuild automatically several rpm in chroot, given a sourcerpm repository, and mail authors or rebuilder when problems occurs.", 
@@ -77,9 +77,24 @@ $run{todo} = [ ];
     [ "", "distro", 1, "<distro>", 
     "Set the distribution",
     sub { ($run{distro}) = @_; 1 }, "Setting the distribution" ],
-    [ "", "dkms", 0, "", 
+    [ "", "dkms", [                                                                                                                                     
+        ["", "dkms", 0, "",                                  
+	"Set the DKMS rebuild mode",
+        sub {    
+            my ($tmp, @arg) = @_; 
+            $tmp->[0] ||= {}; 
+            push @$tmp, @arg; 
+            1 
+        }, "Setting auto mode arguments"], 
+        ["k", "kmedia", 1, "<kernel media regexp>",  
+        "Media Regexp to limit the kernel search to",  
+        sub { my ($tmp, $kmedia) = @_; $tmp->[0]{kmedia} = $kmedia; 1 }, "Limiting rebuild to the kernel in the given media regexp"], 
+        ["m", "media", 1, "<media regexp>",  
+        "Media Regexp to limit rebuild to",  
+        sub { my ($tmp, $media) = @_; $tmp->[0]{media} = $media; 1 }, "Limiting rebuild to the given media regexp"], 
+], "[options]",  
     "Set the DKMS rebuild mode",
-    sub { ($run{dkms}) = 1; 1 }, "Running a DKMS rebuild run" ],
+    sub { my ($opt) = @_; $run{dkms} = $opt; 1 }, "Running a DKMS rebuild run" ],
     [ "a", "arch", 1, "<architecture>", 
     "Set the architecture",
     sub { ($run{my_arch}) = @_; 1 }, "Setting architecture" ],
@@ -297,15 +312,40 @@ if ($run{distro} ne 'cooker') {
 
 -d $config->{upload} or usage($program_name, \@params, "$config->{upload} does not exist");
 
+my $lock = $run{media};
+my $local;
+if (!$lock && $run{chroot}) {
+    $lock = 'chroot';
+    $local = 1
+}
+if (!$lock && $run{dkms}) {
+    $lock = 'dkms';
+    $local = 0
+}
+$run{lock} = $lock;
 # cache file name is needed early to remove the manual lock file if the lock mechanism does not work
-my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$run{media}.cache";
+my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$lock.cache";
 $run{cachefile} = $cachefile;
 if (!$run{debug} && $run{media} || $run{chroot}) {
     $run{pidfile_home} = "$config->{cache_home}/";
-    $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$run{media}";
+    $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$lock";
     check_pid(\%run)
 }
 
+$config->{local_upload} ||= $config->{local_home};
+my $local_spool;
+if ($config->{local_spool}) {
+    $local_spool = $config->{local_spool}
+} else {
+    $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/"
+}
+print {$run{LOG}} "$program_name: using $local_spool as local spooler\n" if $run{verbose} > 4;
+if (!-d "$local_spool/log") {
+    print {$run{LOG}} "$program_name: creating local spool $local_spool\n" if $run{verbose} > 4;
+    mkdir_p("$local_spool/log") or die "FATAL $program_name: could not create local spool dir $local_spool ($!)"
+}
+$run{local_spool} = $local_spool;
+
 my $debug_tag = '_debug' if $run{debug};
 $run{debug_tag} = $debug_tag;
 my $chroot_name = "chroot_$run{distro_tag}$debug_tag";
@@ -314,7 +354,7 @@ $run{chroot_path} = $chroot;
 my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
 $run{chroot_tar} = $chroot_tar;
 if ($run{chroot} || !-d "$chroot/dev") {
-    check_chroot($chroot, $chroot_tar, \%run) or die "FATAL $program_name: could not prepare initial chroot"
+    check_build_chroot($chroot, $chroot_tar, \%run, $config) or die "FATAL $program_name: could not prepare initial chroot"
 }
 
 my $cache;
@@ -415,7 +455,7 @@ $chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}/$chroot_name.$run{run
 $run{chroot_tmp} = $chroot_tmp;
 
 # now exit if there is nothing to do and it was just a cleaning pass
-if ($run{no_compile} || !@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild} && !$run{dkms}) {
+if ($run{no_compile} || !@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild}) {
     send_status_mail(\%run, $config, $cache) if ($run{status_mail});
     print {$run{LOG}} "$program_name: no package to compile :(\n";
     unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile};
@@ -431,9 +471,9 @@ if ($df->{per} == 100) {
 
 if ($run{shell}) {
     ($union_id, my $chroot_tmp) = create_temp_chroot(\%run, $config, $cache, $union_id) or die "FATAL $program_name: could not create temporary chroot";
-    add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL $program_name: could not add local user";
+    add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or die "FATAL $program_name: could not add local user";
     $urpmi->set_command($chroot_tmp);
-    $urpmi->add_packages($chroot_tmp, $luser, 'sudo', 'urpmi') or die "FATAL $program_name: could not add urpmi and sudo in the chroot";
+    $urpmi->install_packages('chroot', $chroot_tmp, $local_spool, \%pack_provide, 'configure', "[ADMIN] installation of urpmi and sudo failed in the chroot $run{my_arch}", $config->{admin}, 'urpmi', 'sudo') or die "FATAL $program_name: could not add urpmi and sudo in the chroot";
     add_sudoers(\%run, $chroot_tmp, $luser);
     if ($run{shell}) {
 	print {$run{LOG}} "$program_name: dumping to a chrooted shell into $chroot_tmp\n";
@@ -442,19 +482,6 @@ if ($run{shell}) {
     }
 }
 
-$config->{local_upload} ||= $config->{local_home};
-my $local_spool;
-if ($config->{local_spool}) {
-    $local_spool = $config->{local_spool}
-} else {
-    $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/"
-}
-print {$run{LOG}} "$program_name: using $local_spool as local spooler\n" if $run{verbose} > 4;
-if (!-d "$local_spool/log") {
-    print {$run{LOG}} "$program_name: creating local spool $local_spool\n" if $run{verbose} > 4;
-    mkdir_p("$local_spool/log") or die "FATAL $program_name: could not create local spool dir $local_spool ($!)"
-}
-
 # perform some cleaning before running to have some more space, rsync to the server too in case previous iurt crashed
 if ($config->{rsync_to} && !$run{no_rsync}) { 
     	# remove some old and very big log files not to saturate the server
@@ -462,14 +489,7 @@ if ($config->{rsync_to} && !$run{no_rsync}) {
 	system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/");
 }
 
-print {$run{LOG}} "$program_name: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3;
-if (!dump_rpmmacros(\%run, $config, "$chroot/home/builder/.rpmmacros")) {
-     print {$run{LOG}} "ERROR $program_name: could not dump rpm macros to $chroot/home/builder/.rpmmacros, trying to rebuild the chroot\n";
-     check_chroot($chroot, $chroot_tar, \%run) or die "FATAL $program_name: could not prepare initial chroot";
-     dump_rpmmacros(\%run, $config, "$chroot/home/builder/.rpmmacros") or die "FATAL $program_name: could not dump rpm macros to $chroot/home/builder/.rpmmacros"
-}
-
-if ($run{dkms}) {
+if ($run{dkms} && $run{dkms_todo}) {
     $done += $dkms->dkms_compile($local_spool, $done);
 }
 
@@ -537,7 +557,7 @@ do {
 	#($maintainer, $cc) = ($config->{admin},'');
 
 	print {$run{LOG}} "$program_name: adding local user $luser into $chroot_tmp...\n" if $run{verbose};
-	add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or next;
+	add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or next;
 
 	my $ret = recreate_srpm(\%run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry);
 	if ($ret == -1) {
@@ -548,11 +568,11 @@ do {
 	} 
 	
 	print {$run{LOG}} "$program_name: installing build dependencies of $srpm...\n" if $run{verbose};
-	my $path_srpm =  $run{chrooted_urpmi} ? "/home/$luser/rpm/SRPMS/" : "$chroot_tmp/home/$luser/rpm/SRPMS/";
+	my $path_srpm = "$chroot_tmp/home/$luser/rpm/SRPMS/";
 	# 
 	# on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore 
 	# installed packages, retrying several time to be sure something is really broken
-	my $ok = $urpmi->install_srpm_deps_hard($path_srpm, $srpm, $chroot_tmp, $local_spool, \%pack_provide, $maintainer);
+	my $ok = $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'install_deps', "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", $maintainer, "$path_srpm/$srpm");
 	if (!$ok) {
 	    $run{status}{$srpm} = 'install_deps_failure';
 	    next;
@@ -569,13 +589,13 @@ do {
 	print {$run{LOG}} "Compiling $srpm\n" if $run{verbose}; 
 	my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm";
 	if ($run{stop}) {
-	    $urpmi->add_packages($chroot_tmp, $luser, 'urpmi', 'sudo');
+	    $urpmi->install_packages('chroot', $chroot_tmp, $local_spool, \%pack_provide, 'configure', "[ADMIN] installation of urpmi and sudo failed in the chroot $run{my_arch}", $config->{admin}, 'urpmi', 'sudo');
 	    add_sudoers(\%run, $chroot_tmp, $luser);
 	    $command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec"
 	}
 	if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, 
 		\%run, $config, $cache, 
-		#	mail => $maintainer, 
+		mail => $maintainer, 
 		error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", 
 		hash => "build_$srpm", 
 		timeout => 18000, 
@@ -606,8 +626,8 @@ do {
 			my @rpm = find_provides(\%run, \%pack_provide, $missing);
 			print {$run{LOG}} "$program_name: likely @rpm ($missing-devel) needed to rebuilt $srpm is not in build_requires\n" if $run{verbose} > 4;
 			if ($maintainer ne 'NOT_FOUND') {
-			    $opt->{mail} = $config->{admin};
-			    #$opt->{mail} .= ", $other_maint";
+			    $opt->{mail} = $maintainer;
+			    #$opt->{mail} .= ", other_maint";
 			}
 			if (!$opt->{mail}) { 
 			    $opt->{mail} = $config->{admin};
@@ -659,10 +679,10 @@ do {
 	foreach my $bindir (readdir $binfh) {
 	    -d "$chroot_tmp/home/$luser/rpm/RPMS/$bindir" or next;
 	    opendir my $rpmfh, "$chroot_tmp/home/$luser/rpm/RPMS/$bindir";
-	    push @packages, map { "/home/$luser/rpm/RPMS/$bindir/$_" } grep { /\.rpm$/ } readdir $rpmfh;
+	    push @packages, map { "$chroot_tmp/home/$luser/rpm/RPMS/$bindir/$_" } grep { !/src\.rpm$/ && /\.rpm$/ } readdir $rpmfh;
 	}
 	# We should fail here, but rpm is currently segfaulting when trying to install packages
-	$urpmi->install_packages($local_spool, $srpm, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", @packages) ;# or next;
+	$urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", @packages);# or next;
 	
 	$run{status}{$srpm} = 'ok';
 	delete $cache->{failure}{$srpm} if defined $cache->{failure}{$srpm};
@@ -831,32 +851,8 @@ sub check_version {
 	0
 }
 
-sub check_chroot {
-    my ($chroot, $chroot_tar, $run, $opt) = @_;
-    print {$run{LOG}} "$program_name: checking basesystem tar\n" if $run{verbose};
-    my $urpmi = $run->{urpmi};
-    $urpmi->clean_urpmi_process($chroot);
-    if (!clean_chroot($chroot, $chroot_tar, $run, $config, 1)) {
-	print {$run{LOG}} "ERROR $program_name: Could no prepare initial chroot";
-	return
-    }
-    my (@stat) = stat $chroot_tar;
-    if (time - $stat[9] > 604800) {
-	print {$run{LOG}} "$program_name check_chroot: tar chroot is more than one week old, forcing the rebuild\n";
-	system("sudo rm -rf $chroot_tar $chroot");
-    }
-    perform_command("sudo $config->{install_chroot_binary} $config->{basesystem_media_root} $config->{basesystem_media_root}/media/$config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build rpm-mandriva-setup-build", 
-	$run, $config, $cache, 
-	mail => $config->{admin},
-	error => "[REBUILD] Creating the inital chroot for $run->{distro_tag} on $run->{my_arch} failed", 
-	hash => 'chroot_inititialization', 
-	timeout => 600, 
-	debug_mail => $run->{debug},
-	die => 1);
-}
-
 sub check_pid {
-    my ($run) = @_;
+    my ($run, $local) = @_;
     my $hostname = `hostname`;
     chomp $hostname;
     my $pidfile = $run->{pidfile};
@@ -877,7 +873,7 @@ sub check_pid {
 	    close $lock2;
 	}
     }
-    if (!$run->{concurrent_run}) {
+    if (!$run->{concurrent_run} && !$local) {
 	opendir my $dir, $run->{pidfile_home};
 	foreach my $f (readdir $dir) {
 	    my ($pid_host) = $f =~ /$pidfile\.pid\.(.*)\.pid$/ or next; 
@@ -897,7 +893,6 @@ sub check_pid {
 		}
 	    }
 	}
-	$run->{pidfile} .= ".pid";
     }
     $run->{pidfile} .= ".$hostname.pid";
     $pidfile = "$run->{pidfile_home}/$run->{pidfile}";
@@ -1111,7 +1106,7 @@ sub recreate_srpm {
 	    \%run, $config, $cache, %opt)) {
 	print {$run->{LOG}} "ERROR $program_name: chrooting failed (retry $opt{retry}\n" if $run->{debug};
 	if ($opt{retry}) {
-	    check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) or return;
+	    check_build_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run,  $config) or return;
 	    return -1
 	}
 	return
@@ -1212,7 +1207,7 @@ sub send_status_mail {
 	}
     }
     print "$text\n";
-    sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", 0, "Iurt", 0)
+    sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", "Iurt the rebuild bot <$config->{admin}>", 0)
 }
 
 sub find_provides {