diff options
Diffstat (limited to 'iurt2')
| -rwxr-xr-x | iurt2 | 202 |
1 files changed, 145 insertions, 57 deletions
@@ -47,7 +47,7 @@ $run{todo} = [ ]; @params = ( # [ "one letter option", "long name option", "number of args (-X means ´at least X´)", "help text", "function to call", "log info"] [ "", "$program_name", 0, "[--cache] [--concurrent-run] [--config foo value] [--warn] [--verbose integer] - [--copy_srpm] [--debug] [--distro] [--no_rsync] [--stop {p|c|i|l|b|a|s}] + [--copy_srpm] [--debug] [--distro] [--no_rsync] [--shell] [--stop {p|c|i|l|b|a|s}] [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--unionfs] [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs] {--config_help | @@ -152,9 +152,17 @@ $run{todo} = [ ]; [ "w", "warn", 0, "", "Warn maintainer of the packages about problem in the rebuild", sub { $run{warn} = 1 ; 1 }, "Setting warn flag to warn maintainers" ], + [ "", "shell", 0, "", + "Dump to a shell into the newly created chroot with sudo on rpm, urpmi, urpme and urpmi.addmedia", + sub { + ($run{shell}) = 1; + 1 }, "Setting option to dump to a shell" ], [ "", "stop", 1, "<rpm step>", "Perform rpm -b<rpm step> (p c i l b a s) instead of rpm -ba and then open a shell in the chroot", - sub { ($run{stop}) = @_; 1 }, "Setting rpm build option" ], + sub { + ($run{stop}) = @_; + 1 + }, "Setting rpm build option" ], ); open(my $LOG, ">&STDERR"); @@ -296,7 +304,7 @@ $to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{ext dump_cache(\%run); -if (!@{$run{todo}} && !$run{debug}) { +if (!@{$run{todo}} && !$run{debug} && !$run{shell}) { print {$run{LOG}} "iurt: nothing to do\n"; unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile}; exit @@ -316,14 +324,17 @@ print {$run{LOG}} "iurt: using $run{run} as chroot extension\n" if $run{verbose} $config->{local_upload} ||= $config->{local_home}; my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}"; -if (!-d "$config->{local_upload}/iurt/$run{distro_tag}/") { +if (!-d $local_spool) { + print {$run{LOG}} "iurt: creating local spool $local_spool\n" if $run{verbose} > 4; -d "$config->{local_upload}/iurt" or mkdir "$config->{local_upload}/iurt"; - mkdir "$config->{local_upload}/iurt/$run{distro_tag}"; + my $d = "$config->{local_upload}/iurt/$run{distro_tag}"; + if (!-d $d) { mkdir $d or die "FATAL iurt: could not create local spool dir $d ($!)" } if (!-d $local_spool) { mkdir $local_spool; mkdir "$local_spool/log" } } + # perform some cleaning before running to have some more space, rsync to the server too in case previous iurt crashed if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server @@ -331,6 +342,7 @@ if ($config->{rsync_to} && !$run{no_rsync}) { system("rsync --delete -alHPe 'ssh -c arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/"); } +print {$run{LOG}} "iurt: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3; if (!dump_rpmmacros("$chroot/home/builder/.rpmmacros")) { print "ERROR iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros, trying to rebuild the chroot"; check_chroot($chroot, $chroot_tar, \%run); @@ -386,6 +398,21 @@ if ($df->{per} == 100) { die "FATAL iurt: not enough space on the filesystem, only $df->{bavail}KB on $home, full at $df->{per}%" } +$run{user} = $ENV{SUDO_USER} || $ENV{USER}; +$run{uid} = getpwnam $run{user}; +print {$run{LOG}} "iurt: using local user $run{user}, id $run{uid}\n" if $run{verbose} > 3; +my $luser = $run{user} || 'builder'; + +if ($run{shell}) { + my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id); + add_local_user(\%run, $luser, $run{uid}); + add_sudo($chroot_tmp, $luser); + if ($run{shell}) { + print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n"; + exec "sudo chroot $chroot_tmp /bin/su $luser -c bash" + } +} + foreach (my $i ; $i < @{$run{todo}}; $i++) { my ($dir, $srpm) = @{$run{todo}[$i]}; $done{$srpm} and next; @@ -403,36 +430,8 @@ retry: dump_cache(\%run); die "FATAL iurt: Could not have urpmi working !" } - if ($unionfs_tmp) { - my $mount_point = "$unionfs_dir/unionfs.$run{run}.$union_id"; - print {$run{LOG}} "Cleaning $mount_point\n" if $run{verbose} > 1; - if (!clean_mnt($mount_point, $run{verbose})) { - dump_cache(\%run); - die "FATAL iurt: could not kill remaining processes acceding $mount_point" - } - my $tmpfs; - - # we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database) - $union_id = clean_unionfs($unionfs_dir, $run{run}, $union_id); - $tmpfs = "$unionfs_dir/tmpfs.$run{run}.$union_id"; - $chroot_tmp = "$unionfs_dir/unionfs.$run{run}.$union_id"; - mkdir $tmpfs or die "Could not create $tmpfs ($!)"; - mkdir $chroot_tmp or die "Could not create $chroot_tmp ($!)"; - if ($cache->{no_unionfs}{$srpm}) { - $unionfs_tmp = 0; - clean_chroot($chroot_tmp, \%run) - } else { - # if the previous package has been built without unionfs, chroot need to be cleaned - clean_chroot($chroot_tmp, \%run) if !$unionfs_tmp; - $unionfs_tmp = 1; - system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null}) and die "FATAL iurt: could not mount $tmpfs ($!)"; - system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|) and die "FATAL iurt: could not mount $tmpfs and $home/chroot_$run{distro_tag}$debug_tag with unionfs ($!)"; - system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null") and die "FATAL iurt: could not mount /proc in the chroot $chroot_tmp."; - } - } else { - print {$run{LOG}} "iurt: installing a new chroot for $srpm in $chroot_tmp\n" if $run{verbose} > 1; - clean_chroot($chroot_tmp, \%run) - } + my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm); + my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; my ($maintainer, $cc); if (!$run{warn}) { @@ -445,10 +444,12 @@ retry: } } #($maintainer, $cc) = ($config->{admin},''); - + + add_local_user(\%run, $luser, $run{uid}); + # recreate a new srpm for buildarch condition in the spec file print {$run{LOG}} "Copying $srpm to $chroot_tmp\n" if $run{verbose} > 1; - perform_command("sudo cp $dir/$srpm $chroot_tmp/home/builder/rpm/SRPMS/", + perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/", \%run, $config, mail => $config->{admin}, error => "[REBUILD] cannot copy $srpm to $chroot_tmp", @@ -459,26 +460,31 @@ retry: error => "[REBUILD] cannot install $srpm in $chroot_tmp", debug_mail => $run{debug}, hash => "install_$srpm", + retry => $retry, callback => sub { my ($opt, $output) = @_; print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug}; - if ($output =~ /user builder does not exist/) { + if ($output =~ /user $luser does not exist/) { print {$run{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run{verbose} > 1; $opt->{error} = "[CHROOT] chroot is corrupted"; - $opt->{retry} = 1 if !$retry + $opt->{retry} = 1 if !$opt->{retry}; + return } + 1 } ); - if (!perform_command(qq{sudo chroot $chroot_tmp su builder -c "rpm -i /home/builder/rpm/SRPMS/$srpm"}, + if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"}, \%run, $config, %opt)) { + print {$run{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run{debug}; if ($opt{retry}) { $retry = 1; check_chroot($chroot, $chroot_tar, \%run); goto retry } + die; next } - perform_command(qq{sudo chroot $chroot_tmp su builder -c "rpm --nodeps -bs /home/builder/rpm/SPECS/*.spec"}, + perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm --nodeps -bs /home/$luser/rpm/SPECS/*.spec"}, \%run, $config, mail => $config->{admin}, error => "[REBUILD] cannot create $srpm in $chroot_tmp", @@ -486,7 +492,7 @@ retry: hash => "create_$srpm") or next; print {$run{LOG}} "Installing build dependencies of $srpm...\n" if $run{verbose} > 1; - if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/builder/rpm/SRPMS/$srpm", + if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm", \%run, $config, mail => $config->{admin}, error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", @@ -537,11 +543,12 @@ retry: debug_mail => $run{debug}, log => "$local_spool/log/$srpm/"); # or next; As this failed quite often, do not stop print {$run{LOG}} "Compiling $srpm\n" if $run{verbose}; - my $command = "rpm --rebuild /home/builder/rpm/SRPMS/$srpm"; + my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm"; if ($run{stop}) { - $command = "rpm -b$run{stop} /home/builder/rpm/SPECS/*.spec" + add_sudo($chroot_tmp, $luser); + $command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec" } - if (!perform_command(qq{TMP=/home/builder/tmp/ sudo chroot $chroot_tmp /bin/su builder -c "$command"}, + if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, \%run, $config, mail => $maintainer, error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", @@ -555,7 +562,8 @@ retry: callback => sub { my ($opt, $output) = @_; if ($run{stop}) { - exec "sudo chroot $chroot_tmp /bin/su builder -c bash" + print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n"; + exec "sudo chroot $chroot_tmp /bin/su $luser -c bash" } print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug}; if ($unionfs_tmp && $output =~ /no space left on device/i) { @@ -568,7 +576,7 @@ retry: return 1 } }, - freq => 1) && !glob "$chroot_tmp/home/builder/rpm/RPMS/*/*.rpm") { + freq => 1) && !glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") { # FIXME # The simple algo used here is : # try to compile it with unionfs, if it runs out of space, compile it without the next time @@ -594,7 +602,7 @@ retry: } # do some cleaning if the compilation is successful delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm}; - if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/builder/rpm/RPMS/*/*.rpm", + if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", \%run, $config, mail => $maintainer, error => "[REBUILD] binaries packages generated from $srpm do not install correctly", @@ -615,9 +623,9 @@ retry: } else { print {$run{LOG}} "iurt: build successful, copying packages to $local_spool.\n"; $run{status}{$srpm} = 'ok'; - system("cp $chroot_tmp/home/builder/rpm/RPMS/*/*.rpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/builder/rpm/RPMS/ to $local_spool ($!)\n"; + system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_spool ($!)\n"; if ($run{copy_srpm}) { - system("cp $chroot_tmp/home/builder/rpm/SRPMS/$srpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/builder/rpm/SRPMS/ to $local_spool ($!)\n"; + system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_spool ($!)\n"; } process_queue($config, \%run, \@wrong_rpm, 1) } @@ -812,7 +820,7 @@ sub process_queue { my ($config, $run, $wrong_rpm, $quiet) = @_; return if !$run->{upload} && $quiet; my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}"; - opendir my $rpmdir, $dir or next; + opendir my $rpmdir, $dir or return; foreach my $rpm (readdir $rpmdir) { my ($rarch, $srpm) = update_srpm($dir, $rpm, $wrong_rpm); $rarch or next; @@ -888,7 +896,7 @@ sub dump_cache { if (-f "$filename.lock") { print {$run{LOG}} "ERROR iurt: manual file lock exist, do not save the cache\n"; } else { - open my $lock, "$filename.lock"; + open my $lock, ">$filename.lock"; print $lock, $$; close $lock; unlink $filename; @@ -1018,8 +1026,10 @@ sub perform_command { local $/; $output = <$log> } + my $call_ret = 1; if (ref $opt{callback}) { - $opt{callback}(\%opt, $output) and return + $call_ret = $opt{callback}(\%opt, $output); + $call_ret == -1 and return; } if ($kill) { $output = "Command has been killed after $opt{timeout} seconds: $command\n$output"; @@ -1041,7 +1051,7 @@ sub perform_command { } return 0 } - if ($kill || $err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) { + if (!$call_ret || $kill || $err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) { if ($opt{log} && $config->{log_url}) { $output = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$output| } @@ -1114,20 +1124,24 @@ sub check_pid { my $hostname = `hostname`; chomp $hostname; my $pidfile = $run->{pidfile}; - my $lockfile = "$run->{pidfile_home}/$pidfile$hostname.pid.lock"; + my $lockfile = "$run->{pidfile_home}/$pidfile.$hostname.pid.lock"; print {$run->{LOG}} "iurt: trying to lock $lockfile\n"; open my $lock, ">$lockfile"; my $lock_ok; - # lockf seems not to work + # lockf seems not to work, try to workarround, but this start to create lock on the lock for the lock of the file. my $status = 1; #File::lockf::lock($lock); if (!$status) { $lock_ok = 1; } else { print {$run->{LOG}} "ERROR iurt: could not lock pid file (status $status $!)\n"; if (! -f "$lockfile.2") { - open my $lock2, ">$lockfile.2"; + print {$run->{LOG}} "iurt: using $lockfile.2 as lock file\n"; + open my $lock2, ">$lockfile.2" or die "FATAL iurt: could not open lock file $lockfile.2"; + print $lock2 $$; close $lock2; } else { + # protection mecchanism to remove dead lock files + unlink "$lockfile.2"; die "FATAL iurt: could not lock pid file (status $status $!)\n"; } } @@ -1303,3 +1317,77 @@ sub get_date { my $daydate = sprintf "%4d%02d%02d", $year, $mon+1, $mday; $fulldate, $daydate } + +sub add_local_user { + my ($run, $luser, $uid) = @_; + # change the builder user to the local user id + # FIXME it seems that unionfs does not handle well the change of the uid of files + # if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) { + if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|)) { + print {$run->{LOG}} "ERROR: setting userid to builder user in chroot failed, trying to check the chroot\n"; + check_chroot($chroot, $chroot_tar, $run); + } + if (system(qq|sudo chroot $chroot_tmp cp -R /home/builder /home/$luser|)) { + die "FATAL iurt: could not initialized $luser directory\n"; + } + if (system(qq|sudo chroot $chroot_tmp chown -R $uid /home/$luser|)) { + die "FATAL iurt: could not initialized $luser directory\n"; + } +} + +sub create_temp_chroot { + my ($run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm) = @_; + if ($unionfs_tmp) { + my $mount_point = "$unionfs_dir/unionfs.$run->{run}.$union_id"; + print {$run->{LOG}} "Cleaning $mount_point\n" if $run->{verbose} > 1; + if (!clean_mnt($mount_point, $run->{verbose})) { + dump_cache($run); + die "FATAL iurt: could not kill remaining processes acceding $mount_point" + } + my $tmpfs; + + # we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database) + $union_id = clean_unionfs($unionfs_dir, $run->{run}, $union_id); + $tmpfs = "$unionfs_dir/tmpfs.$run->{run}.$union_id"; + $chroot_tmp = "$unionfs_dir/unionfs.$run->{run}.$union_id"; + mkdir $tmpfs or die "Could not create $tmpfs ($!)"; + mkdir $chroot_tmp or die "Could not create $chroot_tmp ($!)"; + if ($cache->{no_unionfs}{$srpm}) { + $unionfs_tmp = 0; + clean_chroot($chroot_tmp, $run) + } else { + # if the previous package has been built without unionfs, chroot need to be cleaned + clean_chroot($chroot_tmp, $run) if !$unionfs_tmp; + $unionfs_tmp = 1; + system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null}) and die "FATAL iurt: could not mount $tmpfs ($!)"; + system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run->{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|) and die "FATAL iurt: could not mount $tmpfs and $home/chroot_$run->{distro_tag}$debug_tag with unionfs ($!)"; + system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null") and die "FATAL iurt: could not mount /proc in the chroot $chroot_tmp."; + } + } else { + print {$run->{LOG}} "iurt: installing a new chroot in $chroot_tmp\n" if $run->{verbose} > 1; + clean_chroot($chroot_tmp, $run) + } + $unionfs_tmp, $chroot_tmp +} + +sub add_sudo { + my ($chroot, $user) = @_; + my $f; + if (system("sudo urpmi $urpmi_options --root $chroot_tmp sudo urpmi")) { + die "FATAL iurt: could not install sudo in the $chroot_tmp ($!)" + } + my $file = "$chroot/etc/sudoers"; + my $f; + if (!open $f, qq{| sudo sh -c "cat > $file"}) { + print {$run{LOG}} "ERROR iurt: could not open $file ($!)\n"; + return 0 + } + print $f qq{Cmnd_Alias RPM=/bin/rpm,/usr/sbin/urpmi,/usr/sbin/urpme,/usr/sbin/urpmi.addmedia +root ALL=(ALL) ALL +$user ALL=(ALL) NOPASSWD:RPM +}; + close $f; + print {$run{LOG}} "iurt: adding sudo for /bin/rpm, /usr/sbin/urpmi and /usr/sbin/urpme\n"; + -f $file or return 0; + 1 +} |