From 5ab191a5aaf782ecb8f5e1600815847bf58f0b5b Mon Sep 17 00:00:00 2001 From: Florent Villard Date: Mon, 28 Aug 2006 16:35:50 +0000 Subject: - start to use a wrapper for sudo commands - add suboptions to dkms to limit media and kernels - use a special cache file for dkms mode - use the new check_build_chroot functions - use urpmi->install_packages systematically - do not add macros on the initial chroot, it will be done later - move check_chroot functions to Iurt::Chroot --- iurt2 | 133 ++++++++++++++++++++++++++++++++---------------------------------- 1 file changed, 64 insertions(+), 69 deletions(-) (limited to 'iurt2') diff --git a/iurt2 b/iurt2 index 0e73796..b8acbf1 100755 --- a/iurt2 +++ b/iurt2 @@ -36,7 +36,7 @@ use Data::Dumper; use URPM; use Iurt::DKMS; use Iurt::Urpmi; -use Iurt::Chroot qw(clean_chroot dump_rpmmacros add_local_user create_temp_chroot remove_chroot clean_all_chroot_tmp clean_unionfs clean_chroot_tmp); +use Iurt::Chroot qw(clean_chroot add_local_user create_temp_chroot remove_chroot clean_all_chroot_tmp clean_unionfs clean_chroot_tmp check_build_chroot); use Iurt::Process qw(perform_command clean kill_for_good); use Iurt::Mail qw(sendmail); use File::NCopy qw(copy); @@ -69,7 +69,7 @@ $run{todo} = [ ]; [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}] [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs] [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs] [--status] [--ignore-failure] - {--config_help | --dkms + {--config_help | --dkms {--media } --chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } | --rebuild {cooker|2006.0|community/2006.0|...} {i586|x86_64|ppc|...} {filename1.src.rpm} {filename2.src.rpm} ... {filenamen.src.rpm} }", "$program_name is a perl script to rebuild automatically several rpm in chroot, given a sourcerpm repository, and mail authors or rebuilder when problems occurs.", @@ -77,9 +77,24 @@ $run{todo} = [ ]; [ "", "distro", 1, "", "Set the distribution", sub { ($run{distro}) = @_; 1 }, "Setting the distribution" ], - [ "", "dkms", 0, "", + [ "", "dkms", [ + ["", "dkms", 0, "", + "Set the DKMS rebuild mode", + sub { + my ($tmp, @arg) = @_; + $tmp->[0] ||= {}; + push @$tmp, @arg; + 1 + }, "Setting auto mode arguments"], + ["k", "kmedia", 1, "", + "Media Regexp to limit the kernel search to", + sub { my ($tmp, $kmedia) = @_; $tmp->[0]{kmedia} = $kmedia; 1 }, "Limiting rebuild to the kernel in the given media regexp"], + ["m", "media", 1, "", + "Media Regexp to limit rebuild to", + sub { my ($tmp, $media) = @_; $tmp->[0]{media} = $media; 1 }, "Limiting rebuild to the given media regexp"], +], "[options]", "Set the DKMS rebuild mode", - sub { ($run{dkms}) = 1; 1 }, "Running a DKMS rebuild run" ], + sub { my ($opt) = @_; $run{dkms} = $opt; 1 }, "Running a DKMS rebuild run" ], [ "a", "arch", 1, "", "Set the architecture", sub { ($run{my_arch}) = @_; 1 }, "Setting architecture" ], @@ -297,15 +312,40 @@ if ($run{distro} ne 'cooker') { -d $config->{upload} or usage($program_name, \@params, "$config->{upload} does not exist"); +my $lock = $run{media}; +my $local; +if (!$lock && $run{chroot}) { + $lock = 'chroot'; + $local = 1 +} +if (!$lock && $run{dkms}) { + $lock = 'dkms'; + $local = 0 +} +$run{lock} = $lock; # cache file name is needed early to remove the manual lock file if the lock mechanism does not work -my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$run{media}.cache"; +my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$lock.cache"; $run{cachefile} = $cachefile; if (!$run{debug} && $run{media} || $run{chroot}) { $run{pidfile_home} = "$config->{cache_home}/"; - $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$run{media}"; + $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$lock"; check_pid(\%run) } +$config->{local_upload} ||= $config->{local_home}; +my $local_spool; +if ($config->{local_spool}) { + $local_spool = $config->{local_spool} +} else { + $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/" +} +print {$run{LOG}} "$program_name: using $local_spool as local spooler\n" if $run{verbose} > 4; +if (!-d "$local_spool/log") { + print {$run{LOG}} "$program_name: creating local spool $local_spool\n" if $run{verbose} > 4; + mkdir_p("$local_spool/log") or die "FATAL $program_name: could not create local spool dir $local_spool ($!)" +} +$run{local_spool} = $local_spool; + my $debug_tag = '_debug' if $run{debug}; $run{debug_tag} = $debug_tag; my $chroot_name = "chroot_$run{distro_tag}$debug_tag"; @@ -314,7 +354,7 @@ $run{chroot_path} = $chroot; my $chroot_tar = "$chroot.$run{my_arch}.tar.gz"; $run{chroot_tar} = $chroot_tar; if ($run{chroot} || !-d "$chroot/dev") { - check_chroot($chroot, $chroot_tar, \%run) or die "FATAL $program_name: could not prepare initial chroot" + check_build_chroot($chroot, $chroot_tar, \%run, $config) or die "FATAL $program_name: could not prepare initial chroot" } my $cache; @@ -415,7 +455,7 @@ $chroot_tmp = "$config->{local_home}/chroot_tmp/$run{user}/$chroot_name.$run{run $run{chroot_tmp} = $chroot_tmp; # now exit if there is nothing to do and it was just a cleaning pass -if ($run{no_compile} || !@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild} && !$run{dkms}) { +if ($run{no_compile} || !@{$run{todo}} && !$run{debug} && !$run{shell} && !$run{rebuild}) { send_status_mail(\%run, $config, $cache) if ($run{status_mail}); print {$run{LOG}} "$program_name: no package to compile :(\n"; unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile}; @@ -431,9 +471,9 @@ if ($df->{per} == 100) { if ($run{shell}) { ($union_id, my $chroot_tmp) = create_temp_chroot(\%run, $config, $cache, $union_id) or die "FATAL $program_name: could not create temporary chroot"; - add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL $program_name: could not add local user"; + add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or die "FATAL $program_name: could not add local user"; $urpmi->set_command($chroot_tmp); - $urpmi->add_packages($chroot_tmp, $luser, 'sudo', 'urpmi') or die "FATAL $program_name: could not add urpmi and sudo in the chroot"; + $urpmi->install_packages('chroot', $chroot_tmp, $local_spool, \%pack_provide, 'configure', "[ADMIN] installation of urpmi and sudo failed in the chroot $run{my_arch}", $config->{admin}, 'urpmi', 'sudo') or die "FATAL $program_name: could not add urpmi and sudo in the chroot"; add_sudoers(\%run, $chroot_tmp, $luser); if ($run{shell}) { print {$run{LOG}} "$program_name: dumping to a chrooted shell into $chroot_tmp\n"; @@ -442,19 +482,6 @@ if ($run{shell}) { } } -$config->{local_upload} ||= $config->{local_home}; -my $local_spool; -if ($config->{local_spool}) { - $local_spool = $config->{local_spool} -} else { - $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}/$run{media}/" -} -print {$run{LOG}} "$program_name: using $local_spool as local spooler\n" if $run{verbose} > 4; -if (!-d "$local_spool/log") { - print {$run{LOG}} "$program_name: creating local spool $local_spool\n" if $run{verbose} > 4; - mkdir_p("$local_spool/log") or die "FATAL $program_name: could not create local spool dir $local_spool ($!)" -} - # perform some cleaning before running to have some more space, rsync to the server too in case previous iurt crashed if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server @@ -462,14 +489,7 @@ if ($config->{rsync_to} && !$run{no_rsync}) { system("rsync --delete -alHPe 'ssh -xc arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/"); } -print {$run{LOG}} "$program_name: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3; -if (!dump_rpmmacros(\%run, $config, "$chroot/home/builder/.rpmmacros")) { - print {$run{LOG}} "ERROR $program_name: could not dump rpm macros to $chroot/home/builder/.rpmmacros, trying to rebuild the chroot\n"; - check_chroot($chroot, $chroot_tar, \%run) or die "FATAL $program_name: could not prepare initial chroot"; - dump_rpmmacros(\%run, $config, "$chroot/home/builder/.rpmmacros") or die "FATAL $program_name: could not dump rpm macros to $chroot/home/builder/.rpmmacros" -} - -if ($run{dkms}) { +if ($run{dkms} && $run{dkms_todo}) { $done += $dkms->dkms_compile($local_spool, $done); } @@ -537,7 +557,7 @@ do { #($maintainer, $cc) = ($config->{admin},''); print {$run{LOG}} "$program_name: adding local user $luser into $chroot_tmp...\n" if $run{verbose}; - add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or next; + add_local_user($chroot_tmp, \%run, $config, $luser, $run{uid}) or next; my $ret = recreate_srpm(\%run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry); if ($ret == -1) { @@ -548,11 +568,11 @@ do { } print {$run{LOG}} "$program_name: installing build dependencies of $srpm...\n" if $run{verbose}; - my $path_srpm = $run{chrooted_urpmi} ? "/home/$luser/rpm/SRPMS/" : "$chroot_tmp/home/$luser/rpm/SRPMS/"; + my $path_srpm = "$chroot_tmp/home/$luser/rpm/SRPMS/"; # # on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore # installed packages, retrying several time to be sure something is really broken - my $ok = $urpmi->install_srpm_deps_hard($path_srpm, $srpm, $chroot_tmp, $local_spool, \%pack_provide, $maintainer); + my $ok = $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'install_deps', "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", $maintainer, "$path_srpm/$srpm"); if (!$ok) { $run{status}{$srpm} = 'install_deps_failure'; next; @@ -569,13 +589,13 @@ do { print {$run{LOG}} "Compiling $srpm\n" if $run{verbose}; my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm"; if ($run{stop}) { - $urpmi->add_packages($chroot_tmp, $luser, 'urpmi', 'sudo'); + $urpmi->install_packages('chroot', $chroot_tmp, $local_spool, \%pack_provide, 'configure', "[ADMIN] installation of urpmi and sudo failed in the chroot $run{my_arch}", $config->{admin}, 'urpmi', 'sudo'); add_sudoers(\%run, $chroot_tmp, $luser); $command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec" } if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, \%run, $config, $cache, - # mail => $maintainer, + mail => $maintainer, error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", hash => "build_$srpm", timeout => 18000, @@ -606,8 +626,8 @@ do { my @rpm = find_provides(\%run, \%pack_provide, $missing); print {$run{LOG}} "$program_name: likely @rpm ($missing-devel) needed to rebuilt $srpm is not in build_requires\n" if $run{verbose} > 4; if ($maintainer ne 'NOT_FOUND') { - $opt->{mail} = $config->{admin}; - #$opt->{mail} .= ", $other_maint"; + $opt->{mail} = $maintainer; + #$opt->{mail} .= ", other_maint"; } if (!$opt->{mail}) { $opt->{mail} = $config->{admin}; @@ -659,10 +679,10 @@ do { foreach my $bindir (readdir $binfh) { -d "$chroot_tmp/home/$luser/rpm/RPMS/$bindir" or next; opendir my $rpmfh, "$chroot_tmp/home/$luser/rpm/RPMS/$bindir"; - push @packages, map { "/home/$luser/rpm/RPMS/$bindir/$_" } grep { /\.rpm$/ } readdir $rpmfh; + push @packages, map { "$chroot_tmp/home/$luser/rpm/RPMS/$bindir/$_" } grep { !/src\.rpm$/ && /\.rpm$/ } readdir $rpmfh; } # We should fail here, but rpm is currently segfaulting when trying to install packages - $urpmi->install_packages($local_spool, $srpm, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", @packages) ;# or next; + $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, \%pack_provide, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", @packages);# or next; $run{status}{$srpm} = 'ok'; delete $cache->{failure}{$srpm} if defined $cache->{failure}{$srpm}; @@ -831,32 +851,8 @@ sub check_version { 0 } -sub check_chroot { - my ($chroot, $chroot_tar, $run, $opt) = @_; - print {$run{LOG}} "$program_name: checking basesystem tar\n" if $run{verbose}; - my $urpmi = $run->{urpmi}; - $urpmi->clean_urpmi_process($chroot); - if (!clean_chroot($chroot, $chroot_tar, $run, $config, 1)) { - print {$run{LOG}} "ERROR $program_name: Could no prepare initial chroot"; - return - } - my (@stat) = stat $chroot_tar; - if (time - $stat[9] > 604800) { - print {$run{LOG}} "$program_name check_chroot: tar chroot is more than one week old, forcing the rebuild\n"; - system("sudo rm -rf $chroot_tar $chroot"); - } - perform_command("sudo $config->{install_chroot_binary} $config->{basesystem_media_root} $config->{basesystem_media_root}/media/$config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build rpm-mandriva-setup-build", - $run, $config, $cache, - mail => $config->{admin}, - error => "[REBUILD] Creating the inital chroot for $run->{distro_tag} on $run->{my_arch} failed", - hash => 'chroot_inititialization', - timeout => 600, - debug_mail => $run->{debug}, - die => 1); -} - sub check_pid { - my ($run) = @_; + my ($run, $local) = @_; my $hostname = `hostname`; chomp $hostname; my $pidfile = $run->{pidfile}; @@ -877,7 +873,7 @@ sub check_pid { close $lock2; } } - if (!$run->{concurrent_run}) { + if (!$run->{concurrent_run} && !$local) { opendir my $dir, $run->{pidfile_home}; foreach my $f (readdir $dir) { my ($pid_host) = $f =~ /$pidfile\.pid\.(.*)\.pid$/ or next; @@ -897,7 +893,6 @@ sub check_pid { } } } - $run->{pidfile} .= ".pid"; } $run->{pidfile} .= ".$hostname.pid"; $pidfile = "$run->{pidfile_home}/$run->{pidfile}"; @@ -1111,7 +1106,7 @@ sub recreate_srpm { \%run, $config, $cache, %opt)) { print {$run->{LOG}} "ERROR $program_name: chrooting failed (retry $opt{retry}\n" if $run->{debug}; if ($opt{retry}) { - check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) or return; + check_build_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run, $config) or return; return -1 } return @@ -1212,7 +1207,7 @@ sub send_status_mail { } } print "$text\n"; - sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", 0, "Iurt", 0) + sendmail($run->{status_mail}, '' , "Iurt report for $run->{my_arch}/$run->{media}", "$text", "Iurt the rebuild bot <$config->{admin}>", 0) } sub find_provides { -- cgit v1.2.1