diff options
Diffstat (limited to 'ulri')
| -rwxr-xr-x | ulri | 272 |
1 files changed, 46 insertions, 226 deletions
@@ -18,28 +18,17 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -# compare and rebuild packages on different architecture -# -# TODO -# -# - create a configuration file to handle the various iurt running -# - get the content of the rebuild dir -# - run as many iurt as machines are available and gather results -# - the scheduler just take the results, launch new rebuild, and quit -# - use perl ssh and try to workarround the non working timeout when the -# remote machine is stalled -# - use submitter as packager, not generic name -# use strict; use MDK::Common qw(any cat_ if_ find); -use Iurt::Config qw(config_usage get_date config_init get_author_email check_arch check_noarch get_target_arch get_mandatory_arch); +use Iurt::Config qw(get_date get_author_email get_target_arch get_mandatory_arch); use Iurt::File qw(create_file); -use Iurt::Process qw(check_pid); -use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state); use Iurt::Mail qw(sendmail); +use Iurt::Process qw(check_pid); +use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state load_lock_file_data record_bot_complete schedule_next_retry); +use Iurt::RPM qw(check_arch check_noarch); use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput); -use Iurt::Ulri qw(build_package warn_about_failure); +use Iurt::Ulri qw(build_package fetch_logs_and_cleanup load_config warn_about_failure); use File::Copy 'move'; use File::Path 'make_path'; use File::Temp 'mktemp'; @@ -58,131 +47,7 @@ if (!$ENV{ULRI_LOG_FILE} || !open($LOG, '>>', $ENV{ULRI_LOG_FILE})) { plog_init($program_name, $LOG, 7, 1); -my $HOME = $ENV{HOME}; -my $configfile = "$HOME/.upload.conf"; -my $sysconfigfile = "/etc/iurt/upload.conf"; - -my $config = {}; -foreach my $f ($configfile, $sysconfigfile) { - plog('DEBUG', "load config: $f"); - if (-f $f) { - $config = eval(cat_($f)) - or die "FATAL $program_name: syntax error in $f"; - last; - } -} - -my %config_usage = ( - admin => { - desc => 'mail address of the bot administrator', - default => 'distrib-admin@mandrivalinux.org' - }, - 'arch_translation' => { - desc => "Renaming of arch", - default => { 'sparc64' => 'sparcv9' } - }, - bot => { - desc => "List of bot able to compile the packages", - default => { - i586 => { - localhost => { - iurt => { - user => 'builder', - command => qq(iurt --copy_srpm --group --config local_spool /home/builder/iurt/__DIR__ --no_rsync --chrooted-urpmi -m __MEDIA__ -- http://localhost/distrib/ -p "__PACKAGER__" -r __TARGET__ __ARCH__), - packages => '/home/builder/iurt/', - } , - }, - }, - }, - }, - media => { - desc => 'Corresponding media to add given the current media', - default => { - default => { - "core/backports" => [ "core/backports", "core/release", "core/updates" ], - "core/backports_testing" => [ - "core/backports", "core/backports_testing", - "core/release", "core/updates" - ], - "core/release" => [ "core/release" ], - "core/updates" => [ "core/release", "core/updates" ], - "core/updates_testing" => [ - "core/release", "core/updates", "core/updates_testing" - ], - "nonfree/backports" => [ - "core/backports", "core/release", "core/updates", - "nonfree/backports", "nonfree/release", "nonfree/updates" - ], - "nonfree/backports_testing" => [ - "core/backports", "core/backports_testing", - "core/release", "core/updates", - "nonfree/backports", "nonfree/backports_testing", - "nonfree/release", "nonfree/updates" - ], - "nonfree/release" => [ "core/release", "nonfree/release" ], - "nonfree/updates" => [ - "core/release", "core/updates", - "nonfree/release", "nonfree/updates" - ], - "nonfree/updates_testing" => [ - "core/release", "core/updates", "core/updates_testing", - "nonfree/release", "nonfree/updates", "nonfree/updates_testing" - ], - "tainted/backports" => [ - "core/backports", "core/release", "core/updates", - "tainted/backports", "tainted/release", "tainted/updates" - ], - "tainted/backports_testing" => [ - "core/backports", "core/backports_testing", - "core/release", "core/updates", - "tainted/backports", "tainted/backports_testing", - "tainted/release", "tainted/updates" - ], - "tainted/release" => [ "core/release", "tainted/release" ], - "tainted/updates" => [ - "core/release", "core/updates", - "tainted/release", "tainted/updates" - ], - "tainted/updates_testing" => [ - "core/release", "core/updates", "core/updates_testing", - "tainted/release", "tainted/updates", "tainted/updates_testing" - ], - }, - }, - }, - faildelay => { - desc => "Time after which the rebuild is considered as a failure", - default => 36000 - }, - http_queue => { - desc => 'Address where log can be consulted', - default => 'http://kenobi.mandriva.com/queue ' - }, - queue => { - desc => "Root of the tree where the packages to compile are located", - default => "$HOME/uploads" - }, - tmp => { - desc => "Temporary directory", - default => "$HOME/tmp" - }, - ssh_options => { - desc => "SSH options", - default => "-o ConnectTimeout=5 -o BatchMode=yes -o ServerAliveInterval=5" - }, - packager => { - desc => 'Default packager tag user by bot', - default => 'Mageia Team <http://www.mageia.org>' - }, - 'arch' => { - desc => 'Architectures list for each target', - default => { - default => [ 'i586', 'x86_64' ], - }, - }, -); -config_usage(\%config_usage, $config) if $run{config_usage}; -config_init(\%config_usage, $config, \%run); +my $config = load_config(\%run); my %untranslated_arch; foreach my $k (keys %{$config->{arch_translation}}) { @@ -192,7 +57,7 @@ foreach my $k (keys %{$config->{arch_translation}}) { $run{pidfile_home} = $config->{tmp}; $run{pidfile} = $program_name; -my $pidfile = check_pid(\%run); +my $pidfile = check_pid(\%run, 1); my ($fulldate, $daydate) = get_date(); @@ -238,7 +103,6 @@ foreach my $prefix (keys %pkg_tree) { # TODO: Make this parallel plog('MSG', "check build bot results"); -my %later; my $something_finished; foreach my $prefix (keys %pkg_tree) { my $ent = $pkg_tree{$prefix}; @@ -299,6 +163,7 @@ foreach my $prefix (keys %pkg_tree) { # Everything is fine, build is continuing! # Kill it if that package had failed on a mandatory arch if (check_if_mandatory_arch_failed($media, $ent, $config)) { + plog('INFO', "A mandatory arch had failed, killing the build on $host/$arch"); ssh($remote, "kill -TERM $pid"); $pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch} = 1; create_file("$done_dir/${prefix}_$arch.cancelled", "$bot $host"); @@ -316,23 +181,16 @@ foreach my $prefix (keys %pkg_tree) { } else { plog('FAIL', "$bot died on $host/$arch (status $proc_state), removing lock"); } - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0; - } - - # Either we are done or we should kill the build - plog('INFO', "delete lock file for $prefix"); - unlink $lock_file; + fetch_logs_and_cleanup($remote, $prefix_dir, "$done_dir/$prefix"); + record_bot_complete(\%run, $bot, $arch, $lock_file, $prefix, $ent, $media, $host, $pid); - $run{bot}{$host}{$bot} = 0; - - if (!$status) { - # TODO: fetch/clean the logs next bot; } my $fail; my $later; + my $done; # Check if the build bot finished on the other side # @@ -342,25 +200,31 @@ foreach my $prefix (keys %pkg_tree) { plog('DEBUG', $res); if ($r eq 'install_deps_failure') { plog('FAIL', "install deps failure, rebuild later: $p"); - $later{$prefix} = 1; - $later = 1; - # TODO: fetch/clean the logs - } + if (schedule_next_retry($config, $todo_dir, $prefix, $arch, $pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures})) { + $later = 1; + $pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1; + } else { + plog('FAIL', "Too many retries due to install_deps_failure: $p"); + $fail = 1; + } + } if ($r ne 'ok') { plog('FAIL', "$r: $p"); $fail = 1; + } else { + plog('OK', "build complete: $p"); + $done = 1; } } - if (!$fail && !$later) { + if ($done) { my @list = split "\n", sout($remote, "ls $prefix_dir"); my $error; - my $done; my $arch_check = join '|', $arch, if_($untranslated_arch{$arch}, @{$untranslated_arch{$arch}}); plog('MSG', "checking for $arch_check arch"); foreach my $result (@list) { - $result =~ /\.(src|$arch_check|noarch)\.rpm$/ or next; + $result =~ /\.rpm$/ or next; # do not copy the initial src package $result =~ /^$prefix/ and next; @@ -368,10 +232,6 @@ foreach my $prefix (keys %pkg_tree) { my $result_file = "$done_dir/${prefix}_$result"; plog('OK', "build ok: $result"); - if ($result =~ /\.$arch_check\.rpm$/) { - $done = 1; - } - plog('DEBUG', "copy files to done"); make_path($done_dir); if (sget($remote, "$prefix_dir/$result", @@ -387,40 +247,32 @@ foreach my $prefix (keys %pkg_tree) { # Add the package to the list of built ones, in case we fail another arch and need to cleanup push @{$ent->{rpms}}, $result_file; } - next if $error; + next bot if $error; - if ($done) { - if (check_if_mandatory_arch_failed($media, $ent, $config)) { - # Discard this arch as another mandatory one failed - cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $media, $arch, $config); - } else { - create_file("$done_dir/${prefix}_$arch.done", "$bot $host"); - $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch} = 1; - make_path("$done_dir/$prefix"); - sget($remote, "$prefix_dir/log/*", "$done_dir/$prefix"); - $something_finished = 1; - } - # Either we already fetched the success logs, or don't care - # as this success was discarded due to another failure. + if (check_if_mandatory_arch_failed($media, $ent, $config)) { + # Discard this arch as another mandatory one failed + plog('INFO', "A mandatory arch had failed, discarding the successful build from $host/$arch"); + cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $media, $arch, $config); ssh($remote, "rm -rf $prefix_dir"); - next bot; + } else { + create_file("$done_dir/${prefix}_$arch.done", "$bot $host"); + $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch} = 1; + fetch_logs_and_cleanup($remote, $prefix_dir, "$done_dir/$prefix"); + $something_finished = 1; } } - make_path($fail_dir); + record_bot_complete(\%run, $bot, $arch, $lock_file, $prefix, $ent, $media, $host, $pid); + + # In case of success we have now fetched packages and logs and cleaned up the remote machine + next bot if $done; unless ($pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch}) { - mkdir("$fail_dir/$prefix"); - if (sget($remote, "$prefix_dir/*", "$fail_dir/$prefix")) { - plog('ERROR', "copying from $host:$prefix_dir/ " . - "to $fail_dir/ failed ($!)"); - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0; - } + make_path($fail_dir); + fetch_logs_and_cleanup($remote, $prefix_dir, "$fail_dir/$prefix"); + $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0; } - # clean the log on the compilation machine - ssh($remote, "rm -rf $prefix_dir"); - # We got the logs but want to retry so don't record a failure next bot if $later; @@ -453,8 +305,6 @@ my %to_compile; # crash or just lock ulri somehow foreach my $prefix (sort keys %pkg_tree) { - next if $later{$prefix}; - my $ent = $pkg_tree{$prefix}; my $ready = 1; @@ -521,9 +371,13 @@ foreach my $prefix (sort keys %pkg_tree) { # need to find a bot for each arch foreach my $arch (@$arch_list) { + next if $pkg_tree{$prefix}{media}{$media}{later}{$arch}; + # Skip this arch if the package is already building for it or if it # should not be built on this arch or it has already failed or # succeeded. + # Check again for noarch here, as we may ave triggered a noarch build on + # previous architecture and should not start another one. next if $pkg_tree{$prefix}{media}{$media}{arch}{$arch}; next if $pkg_tree{$prefix}{media}{$media}{arch}{noarch}; next if $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch}; @@ -561,6 +415,7 @@ foreach my $prefix (sort keys %pkg_tree) { "$lock_arch.$bot.$host.$fulldate.$pid.lock"; plog('DEBUG', "create lock $lock_file"); create_file($lock_file, "$program_name $$", time()); + load_lock_file_data(\%{$pkg_tree{$prefix}}, $lock_file, $media, $config); last hosts; } @@ -605,38 +460,3 @@ plog('INFO', "jobs in queue:", %to_compile ? unlink $pidfile; exec "emi" if $something_finished; - -__END__ - -# ulri ends here - -Discussion ----------- - -20060802 (Warly) - -* I prefer creating a separate scheduler, so that it can eventually call - other bots. -* bots should be able to take packages by themselves. -* Iurt will perform several checks, they have to be standard and usable - by the maintainer, the results must be put in a visible directory or path -* We can put packages either in a dir or to prefix all files with the date - and uploader. Having all files in a dir will make the listing simpler. - Prefixing the files could be problematic if we base the rpm name and - version parsing on the filename. -* ulri knows the prefix, he could ask iurt to put the packages in a dir - with the same prefix. - -20060806 (Warly) - -* All the packages are put in done, then the final youri is run to put them - in queue/ - -20061104 (claudio) - -* Instead if having configuration defaults for our environment and using - ulri with the defaults, it would be nicer to have minimalistic/generic - defaults and install a configuration file in kenobi -* Ulri's configuration file could be renamed to .ulri.conf instead of - .upload.conf. ==> ok, it's also used by emi - |