diff options
Diffstat (limited to 'iurt2')
| -rwxr-xr-x | iurt2 | 100 |
1 files changed, 58 insertions, 42 deletions
@@ -50,6 +50,9 @@ $run{todo} = [ ]; [ "c", "chroot", 0, "", "Check chroot and update it if needed", sub { $run{chroot} = 1 }, "Activating debug mode" ], + [ "", "config", 2, "<configuration keyword> <value>", + "Override a configuration file variable", + sub { my ($key, $value) = @_; $run{config}{$key} = $value }, "Overriding configuration variable" ], [ "", "config_help", 0, "", "Explain configuration files keywords", sub { $run{config_usage} = 1 }, "Activating debug mode" ], @@ -64,11 +67,12 @@ $run{todo} = [ ]; sub { $run{log} = pop @_; open $run{LOG}, ">$run{log}" or die "unable to open $run{log}\n"; - print *{$run{LOG}}, "command line: @ARGV\n" + print *{$run{LOG}}, "command line: @ARGV\n"; + 1 }, "Log file" ], [ "m", "media", -1, "", "Media to rebuild", - sub { ($run{media}) = @_ }, "Adding a media to rebuild" ], + sub { ($run{media}) = @_; 1 }, "Adding a media to rebuild" ], [ "r", "rebuild", -2, "<distro> <architecture> <srpm 1> <srpm 2> ... <srpm n>", "Rebuild the packages, e.g. $program_name -r cooker x86_64 /home/foo/rpm/SRPMS/foo-2.3-12mdk.src.rpm", sub { @@ -83,7 +87,8 @@ $run{todo} = [ ]; } else { die "FATAL iurt: $_ does not seems to be a SRPM\n" } - } + } + 1 }, "Activating rebuild mode" ], [ "", "upload", [ ["", "upload", 0, "[options]", @@ -98,15 +103,18 @@ $run{todo} = [ ]; ], "[options]", "Upload the rebuild packages", sub { $run{upload} = 1 }, "Setting the upload flag" ], + [ "", "no_rsync", 0, "", + "Do not send build log to the distant rsync server", + sub { $run{no_rsync} = 1 }, "Setting the no rsync warn flag" ], [ "w", "warn", 0, "", "Warn maintainer of the packages about problem in the rebuild", - sub { $run{warn} = @_ }, "Setting warn flag to warn maintainers" ], + sub { $run{warn} = @_; 1 }, "Setting warn flag to warn maintainers" ], [ "d", "dir", -1, "", "Directory where to find packages to rebuild", - sub { $run{extra_dir} = @_ }, "Adding extra source packages directories" ], + sub { $run{extra_dir} = @_; 1 }, "Adding extra source packages directories" ], ); -open(my $LOG, ">&$run{LOG}"); +open(my $LOG, ">&STDERR"); $run{LOG} = $LOG; my $todo = parseCommandLine("$program_name", \@ARGV, \@params); @@ -116,8 +124,6 @@ foreach my $t (@$todo) { &{$t->[0]}(@{$t->[1]}) or print {$run{LOG}} "ERROR: $t->[2]\n"; } -# maybe time to switch to $program_name::parse_command_line - $run{distro_tag} = $run{distro}; $run{distro_tag} =~ s,/,-,g; my $urpmi_options = "-v --no-verify-rpm --nolock --auto"; @@ -151,6 +157,7 @@ my %config_usage = ( home => { desc => 'Home dir', default => $HOME }, install_chroot_binary => { desc => 'Tool used to create initial chroot', default => 'install-chroot-tar.sh' }, local_home => { desc => 'Where to build packages', default => $HOME }, + local_upload => { desc => 'Where to store build packages and log', default => '' }, log_size_limit => { desc => 'Maximum authorized size for a log file', default => '100M' }, log_size_date => { desc => 'Number of days log should be kept', default => '30' }, minimum_package_number => { "Minimum number of packages in a synthesis file to consider it valid", default => 1000 }, @@ -169,7 +176,7 @@ if (!$arch_comp{$real_arch}{$run{my_arch}}) { die "FATAL iurt: could not compile $run{my_arch} binaries on a $real_arch" } foreach my $k (keys %config_usage) { - $config->{$k} ||= $config_usage{$k}{default} + $config->{$k} ||= $run{config}{$k} || $config_usage{$k}{default} } $config->{upload} .= $run{distro}; @@ -189,7 +196,6 @@ if (!$run{debug} && $run{media} || $run{chroot}) { check_pid(\%run) } - my $debug_tag = '_debug' if $run{debug}; my $chroot = "$config->{local_home}/chroot$debug_tag"; my $chroot_tar = "$chroot-$run{distro_tag}.$run{my_arch}.tar.gz"; @@ -199,41 +205,48 @@ if ($run{chroot}) { my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.cache"; my $cache; -if (-f $cachefile || $run{cache}) { +if (-f $cachefile && $run{cache}) { print {$run{LOG}} "iurt: loading cache file $cachefile\n"; $cache = do $cachefile } else { - $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 0, needed => {}, no_unionfs => {} } + $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {}, no_unionfs => {} } } my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint); +$to_compile = @{$run{todo}}; $to_compile += check_media(\%run, $cache, $config, \%srpm_version, \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media}; $to_compile += search_packages(1, $cache, \%run, @{$run{extra_dir}}) if $run{extra}; dump_cache(); if (!@{$run{todo}} && !$run{debug}) { - print "iurt: nothing to do\n"; + print {$run{LOG}} "iurt: nothing to do\n"; unlink $run{pidfile} if $run{pidfile}; exit } -print "iurt: will try to compile $to_compile packages\n"; +print {$run{LOG}} "iurt: will try to compile $to_compile packages\n"; exit if !$run{rebuild}; -$cache->{run} ||= 1; -$run{run} = ++$cache->{run}; +if ($run{cache}) { + $run{run} = ++$cache->{run} +} else { + my ($sec,$min,$hour,$mday,$mon,$year) = gmtime(time()); + $year += 1900; + my $fulldate = sprintf "%4d%02d%02d%02d%02d%02d", $year, $mon+1, $mday, $hour, $min, $sec; + $run{run} = "0.$fulldate" +} -my $local_spool = "$config->{local_home}/iurt/$run{distro_tag}/$run{my_arch}"; -if (!-d "$config->{local_home}/iurt/$run{distro_tag}/") { - mkdir "$config->{local_home}/iurt/$run{distro_tag}"; +$config->{local_upload} ||= $config->{local_home}; +my $local_spool = "$config->{local_upload}/iurt/$run{distro_tag}/$run{my_arch}"; +if (!-d "$config->{local_upload}/iurt/$run{distro_tag}/") { + mkdir "$config->{local_upload}/iurt/$run{distro_tag}"; if (!-d $local_spool) { mkdir $local_spool; mkdir "$local_spool/log" } } - dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros"; my $unionfs_dir; if ($run{unionfs}) { @@ -243,12 +256,12 @@ if ($run{unionfs}) { } my $s = sub { - print "iurt: dumping cache...\n"; + print {$run{LOG}} "iurt: dumping cache...\n"; dump_cache(); $Data::Dumper::Indent = 0; $Data::Dumper::Terse = 1; - print "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n"; - print "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n"; + print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n"; + print {$run{LOG}} "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n"; exit }; $SIG{TERM} = $s; @@ -267,8 +280,9 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { check_version($srpm) or next; if ($run{debug}) { $run{debug}++ == 2 and exit } $done++; - print "iurt: packages $srpm [$done/$to_compile]\n"; + print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n"; # FIXME unfortunately urpmi stalls quite often +retry: my $match = "urpmi $urpmi_options --root $chroot"; if (!clean_process($match)) { dump_cache(); @@ -318,7 +332,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { #($maintainer, $cc) = ($config->{admin},''); # recreate a new srpm for buildarch condition in the spec file - print "Copying $srpm to $chroot\n"; + print {$run{LOG}} "Copying $srpm to $chroot\n"; perform_command("sudo cp $dir/$srpm $chroot/home/builder/rpm/SRPMS/", \%run, $config, mail => $config->{admin}, @@ -337,16 +351,16 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { error => "[REBUILD] cannot create $srpm in $chroot", debug_mail => $run{debug}, hash => "create_$srpm") or next; - print "Installing build dependencies of $srpm...\n"; + print {$run{LOG}} "Installing build dependencies of $srpm...\n"; perform_command("sudo urpmi $urpmi_options --root $chroot $chroot/home/builder/rpm/SRPMS/$srpm", \%run, $config, - mail => $maintainer, + mail => $config->{admin}, error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", hash => "install_deps_$srpm", timeout => 600, freq => 1, - cc => $cc, + #cc => $cc, debug_mail => $run{debug}, error_regexp => 'cannot be installed', wait_regexp => 'database locked', @@ -355,14 +369,14 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { callback => sub { my ($opt, $output) = @_; print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug}; - my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)| \[(.*)\])?\)/g; + my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g; @missing_deps or return; while (my $missing_deps = shift @missing_deps) { my $version = shift @missing_deps; my $version2 = shift @missing_deps; $version ||= $version2 || 0; my $p = $pack_provide{$missing_deps} || $missing_deps; - my $other_maint = `rpmmon -p $p`; + my $other_maint = `rpmmon -p "$p"`; chomp $other_maint; print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n"; if ($other_maint && $other_maint ne 'NOT_FOUND') { @@ -377,7 +391,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { # try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory # running rpm -qa several time seems to fix the problem foreach (1 .. 3) { - system("sudo chroot $chroot rpm -qa") + system("sudo chroot $chroot rpm -qa &> /dev/null") } perform_command("sudo chroot $chroot rpm -qa", \%run, $config, @@ -385,7 +399,7 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { timeout => 60, debug_mail => $run{debug}, log => "$local_spool/log/"); # or next; As this failed quite often, do not stop - print "Compiling $srpm\n"; + print {$run{LOG}} "Compiling $srpm\n"; if (!perform_command(qq{TMP=/home/builder/tmp/ sudo chroot $chroot /bin/su builder -c "rpm --rebuild /home/builder/rpm/SRPMS/$srpm"}, \%run, $config, mail => $maintainer, @@ -424,6 +438,8 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { $cache->{failure}{$srpm} = 1 } elsif (!$cache->{no_unionfs}{$srpm}) { $cache->{failure}{$srpm} = 1 + } else { + goto retry } next } @@ -443,10 +459,10 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) { next } if ($run{debug}) { - print "iurt: debug mode, skip other packages\n"; + print {$run{LOG}} "iurt: debug mode, skip other packages\n"; exit } else { - system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_spool") and print "ERROR: could not copy rpm files from $chroot/home/builder/rpm/RPMS/ to $local_spool ($!)\n"; + system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_spool") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot/home/builder/rpm/RPMS/ to $local_spool ($!)\n"; process_queue($config, \%run, \@wrong_rpm) } } @@ -463,9 +479,9 @@ if (open my $file, ">$local_spool/log/wrong_srpm_names.log") { } } -if ($config->{rsync_to}) { +if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server - system(qq{find $local_spool/log/ -name "*.log" \( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \) -exec rm -f {} \;}); + system(qq|find $local_spool/log/ -name "*.log" \( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \) -exec rm -f {} \;|); system("rsync --delete -alHPe 'ssh -c arcfour' $local_spool/log/ $config->{rsync_to}/$run{distro_tag}/$run{my_arch}/log/"); } @@ -548,8 +564,8 @@ sub clean { while ($ps = `$cmd "$var"`) { system(qq{$kill_cmd "$var"}); sleep 1; - chomp $ps; - print {$run{LOG}} "Trying to removed previous blocked processes for $var ($ps)\n"; + $ps =~ s/\n/,/g; + print {$run{LOG}} "Trying to remove previous blocked processes for $var ($ps)\n"; return 0 if $i++ > 10 } 1 @@ -621,7 +637,7 @@ sub check_needed { sub process_queue { my ($config, $run, $wrong_rpm) = @_; return if !$run->{upload}; - my $dir = "$config->{local_home}/iurt/$run->{distro_tag}/$run->{my_arch}"; + my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}"; opendir my $rpmdir, $dir or next; foreach my $rpm (readdir $rpmdir) { my ($rarch, $srpm) = update_srpm($dir, $rpm, $wrong_rpm); @@ -753,14 +769,14 @@ sub perform_command { $output = <$log> } my $err = $?; + if (ref $opt{callback}) { + $opt{callback}(\%opt, $output) + } if ($kill) { $output = "Command has been killed after $opt{timeout} seconds: $command\n$output" } else { $output = "Command failed: $command\n$output" } - if (ref $opt{callback}) { - $opt{callback}(\%opt, $output) - } if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/) { $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback}; print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n"; |