diff options
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | NEWS | 91 | ||||
| -rwxr-xr-x | cancel_build | 116 | ||||
| -rwxr-xr-x | emi | 30 | ||||
| -rwxr-xr-x | iurt | 51 | ||||
| -rwxr-xr-x | iurt_root_command | 12 | ||||
| -rw-r--r-- | lib/Iurt/Chroot.pm | 87 | ||||
| -rw-r--r-- | lib/Iurt/Config.pm | 57 | ||||
| -rw-r--r-- | lib/Iurt/Emi.pm | 26 | ||||
| -rw-r--r-- | lib/Iurt/Process.pm | 59 | ||||
| -rw-r--r-- | lib/Iurt/Queue.pm | 112 | ||||
| -rw-r--r-- | lib/Iurt/RPM.pm | 55 | ||||
| -rwxr-xr-x | lib/Iurt/Ulri.pm | 160 | ||||
| -rw-r--r-- | lib/Iurt/Urpmi.pm | 159 | ||||
| -rw-r--r-- | lib/Iurt/Util.pm | 2 | ||||
| -rwxr-xr-x | rebuild_perl_iurt | 42 | ||||
| -rw-r--r-- | t/queue.t | 107 | ||||
| -rwxr-xr-x | ulri | 272 |
18 files changed, 930 insertions, 513 deletions
@@ -1,7 +1,7 @@ NAME=iurt PACKAGE=$(NAME) -VERSION=$(shell egrep -a 'my \$$version = .*' iurt|sed -e "s/';//" -e "s/.*'//") +VERSION=$(shell sed -n -e "s/my \$$version = '\(.*\)'.*/\1/p" iurt) VENDORLIB = $(shell eval "`perl -V:installvendorlib`"; echo $$installvendorlib) INSTALLVENDORLIB = $(DESTDIR)$(VENDORLIB) @@ -30,4 +30,7 @@ TESTS = $(wildcard t/*.t) .PHONY: $(TESTS) $(TESTS): perl -Ilib $@ + perl -Ilib -c ulri + perl -Ilib -c iurt + perl -Ilib -c emi check: $(TESTS) @@ -1,4 +1,93 @@ -- fix timeout message when killing for other reason +- cancel_build: add an utility to cancel a build in the queue + +0.9.0 +- iurt: Sort packages.arch.log +- rebuild_perl_iurt: set --target to correctly build for armv7hl on aarch64 + machines +- iurt: Fix / of the chroot belonging to the user instead of root +- iurt: Make generation of the chroot archive more atomic +- iurt: Use the target directory when reference chroot needs to be updated +- ulri: Limit retries in case of install_deps_failure. A new config option + backoff_delays is a list of delays before retrying, and when reaching the + end of the list, we fail permanently. + +0.8.2.2 +- ulri: Fix a crash after build failures + +0.8.2.1 +- Fix syntax checks to use local tree + +0.8.2 +- Add some tests checking the syntax of iurt/ulri/emi as the main scripts + are not covered by test. +- ulri: Fix syntax +- iurt: Fix build system retrying forever when rpmbuild -br fails to compute + dynamic BuildRequires, for example because some BuildRequires used in %prep + are missing. + +0.8.1 +- iurt: Fix getting a retriable install_deps_failure instead of missing_dep + when some DynamicBuildRequires can't be resolved. +- iurt: Fix buildreqs.nosrc.rpm filename on Mageia build system + +0.8.0 +- iurt: Add support for DynamicBuildRequires +- ulri: Do not remove lock file and mark build as done until we copied the + files. + +0.7.18 +- ulri: Fix false positives of iurt dying on the build machine +- iurt: Fix cleaning chroots (eg: when using --clean-all) + +0.7.17.2 +- iurt: Use consistent log directory (fixes a crash on Mageia BS). + +0.7.17.1 +- iurt: Fix src.rpm name (mga#30344) + +0.7.17 +- iurt: record the list of generated packages +- iurt: allow generated src.rpm to have a different name, some packages use + macros for Name and those can change based on the distro version. +- ulri: rely on status.log to know a build completed rather than having + packages of the right architecture. +- ulri: Fix list of build in progress +- dump_tree: Add cli to get current status + +0.7.16 +- iurt: use --target noarch for noarch packages rather than the arch used to + built them. + +0.7.15 +- iurt: always use urpmi --urpmi-root +- iurt: add arch to all log files + +0.7.14 +- ulri: add architecture in botcmd filename +- iurt: fix updating the chroot + +0.7.13 +- iurt: try again updating packages later + +0.7.12 +- iurt: update packages after adding media rather than before + +0.7.11 +- iurt: allow building arm v5 and v7 on armv8l +- ulri: sort file names in mail +- move check_{no,}arch from Config into a new RPM module. They have nothing + to do with the config and work on an RPM file. +- iurt: update packages after creating chroot + +0.7.10 +- ulri: allow - in host when parsing lock file name +- iurt: give --target to rpmbuild + +0.7.9 +- ulri: call setarch for all 32bit targets rather than only when target is + i586, this allows to build for 32bit arm on aarch64. +- emi: store in the .upload which architectures were uploaded to know if + non mandatory ones are done. 0.7.8.1 - no changes, fixing bad release diff --git a/cancel_build b/cancel_build new file mode 100755 index 0000000..817dfb2 --- /dev/null +++ b/cancel_build @@ -0,0 +1,116 @@ +#!/usr/bin/perl +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +use strict; +use MDK::Common qw(cat_); +use Iurt::Config qw(config_usage get_date config_init); +use Iurt::File qw(create_file); +use Iurt::Process qw(wait_for_lock); +use Iurt::Queue qw(cleanup_failed_build get_upload_tree_state load_lock_file_data); +use Iurt::Util qw(plog_init plog ssh_setup ssh); +use Iurt::Ulri qw(load_config); + +my %run; +my $program_name = 'cancel_build'; +$run{program_name} = $program_name; + +my $LOG; +if (!$ENV{ULRI_LOG_FILE} || !open($LOG, '>>', $ENV{ULRI_LOG_FILE})) { + open($LOG, ">&STDERR"); +} + +plog_init($program_name, $LOG, 7, 1); +my $prefix = shift or die "Usage: $0 <prefix>\n"; + +my $HOME = $ENV{HOME}; +my $configfile = "$HOME/.upload.conf"; +my $sysconfigfile = "/etc/iurt/upload.conf"; + +my $config = load_config(\%run); + +$run{pidfile_home} = $config->{tmp}; +# Use ulri lock as we don't want to run concurrently with it +$run{pidfile} = 'ulri'; +my $pidfile = wait_for_lock(\%run); + +my ($fulldate, $daydate) = get_date(); +$run{daydate} = $daydate; + +($fulldate, $daydate) = get_date(); + +my $todo = "$config->{queue}/todo"; +my $failure = "$config->{queue}/failure"; +my $done = "$config->{queue}/done"; +my $reject = "$config->{queue}/reject"; + +my %pkg_tree = get_upload_tree_state($config); + +if (!defined($pkg_tree{$prefix})) { + plog('ERROR', "Unknown prefix $prefix"); + unlink $pidfile; + exit 1; +} + +my $ent = $pkg_tree{$prefix}; +foreach my $media (keys %{$ent->{media}}) { + foreach my $bot (@{$ent->{media}{$media}{bot}}) { + $run{bot}{$bot->{host}}{$bot->{bot}} = $prefix; + } +} + +foreach my $media (keys %{$ent->{media}}) { + my $path = $ent->{media}{$media}{path}; + my $user = $ent->{user}; + + # Local pathnames + my $done_dir = "$done/$path"; + my $todo_dir = "$todo/$path"; + my $fail_dir = "$failure/$path"; + + # Calling with "noarch" to make it always a fatal failure + plog('INFO', "Failing $prefix and cleaning up done architectures"); + cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $media, "noarch", $config); + foreach my $bot_list (@{$ent->{media}{$media}{bot}}) { + my ($bot, $host, $date, $pid, $arch, $time) = + @$bot_list{qw(bot host date pid arch time)}; + + my $bot_conf = $config->{bot}{$arch}{$host}{$bot}; + my $remote = ssh_setup($config->{ssh_options}, + $bot_conf->{user}, $host); + + my $prefix_dir = "$bot_conf->{packages}/$path/$prefix-$arch/"; + + # If our build is noarch, set arch appropriately. + # + my $lock_file = + "$todo_dir/${prefix}_$arch-noarch.$bot.$host.$date.$pid.lock"; + + if (-f $lock_file) { + plog('DEBUG', "$prefix is noarch"); + $arch = "noarch"; + } else { + $lock_file =~ s/-noarch//; + } + + plog('INFO', "Killing process $pid on $host (building for arch \"$arch\")"); + ssh($remote, "kill -TERM $pid"); + $pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch} = 1; + create_file("$done_dir/${prefix}_$arch.cancelled", "$bot $host"); + } +} + +unlink $pidfile; @@ -20,11 +20,6 @@ # # upload packages in queue when all the mandatory architectures are done # -# TODO -# -# - take the packages in done/ and upload them with youri in queue/ -# - check that the mandatory architectures are present -# # PREFIX : sprintf "$year%02d%02d%02d%02d%02d.$user.$host.${$}_", $mon, $mday, $hour, $min, $sec; use strict; @@ -32,8 +27,8 @@ use Iurt::Config qw(config_usage config_init); use Iurt::Process qw(check_pid); use Iurt::Queue qw(get_upload_tree_state); use Iurt::Util qw(plog_init plog); -use Iurt::Emi qw(find_prefixes_ready_to_upload upload_prefix_in_media); -use MDK::Common qw(cat_ touch); +use Iurt::Emi qw(find_prefixes_ready_to_upload record_uploaded_packages upload_prefix_in_media); +use MDK::Common qw(cat_); my %run; my $program_name = 'emi'; @@ -86,22 +81,10 @@ my %config_usage = ( desc => "Temporary directory", default => "$HOME/tmp" }, - root => { - desc => 'Architecture root dir', - default => "/mnt/BIG/dis/" - }, - upload_user => { - desc => 'User who is uploading packages', - default => 'mandrake' - }, queue => { desc => 'root directory of the various upload queues', default => "$HOME/uploads" }, - ssh_option => { - desc => "SSH options", - default => "-o ConectTimeout=20" - }, ); config_usage(\%config_usage, $config) if $run{config_usage}; @@ -109,7 +92,7 @@ config_init(\%config_usage, $config, \%run); $run{pidfile_home} = $config->{tmp}; $run{pidfile} = "upload"; -my $pidfile = check_pid(\%run); +my $pidfile = check_pid(\%run, 1); my $todo = "$config->{queue}/todo"; my $done = "$config->{queue}/done"; @@ -136,10 +119,11 @@ foreach my $target (keys %targets) { upload_prefix_in_media($config, \%pkg_tree, $prefix, $media, 1); } + # Now that the finishers are done, metadata was updated so the packages are available foreach my $prefix (@{$targets{$target}{$media}{to_upload}}) { - my $path = $pkg_tree{$prefix}{media}{$media}{path}; - touch("$done/$path/$prefix.upload") unless -f "$reject/$path/$prefix.youri"; - } + my $path = $pkg_tree{$prefix}{media}{$media}{path}; + record_uploaded_packages($config, \%pkg_tree, $prefix, $media) unless -f "$reject/$path/$prefix.youri";; + } } } @@ -34,14 +34,15 @@ use strict; use RPM4::Header; -use Iurt::Config qw(config_usage get_date get_prefix config_init get_maint check_arch %arch_comp get_package_prefix); +use Iurt::Config qw(config_usage get_date get_prefix config_init get_maint %arch_comp get_package_prefix); use Data::Dumper; use URPM; use Iurt::Urpmi; use Iurt::Chroot qw(add_local_user create_temp_chroot remove_chroot create_build_chroot clean_chroot); -use Iurt::Process qw(perform_command kill_for_good set_alarm_message sudo); +use Iurt::Process qw(perform_command kill_for_good sudo); use Iurt::Mail qw(sendmail); +use Iurt::RPM qw(check_arch check_noarch); use Iurt::Util qw(plog_init plog); use File::NCopy qw(copy); use File::Path qw(mkpath); @@ -131,7 +132,7 @@ $run{todo} = []; sub { my ($tmp, @media) = @_; $tmp->[0]{media} = \@media; 1 }, "Limiting rebuild to the kernel in the given media regexp"], ] , "[options] <media prefix>", "Create urpmi media inside the chroot instead of using --root (media prefix is like http:///server.mandriva.com/dis/)", - sub { my ($opt, $media) = @_; $opt->{rooted_media} = $media; $run{chrooted_urpmi} = $opt; 1 }, "Activating chroot media" ], + sub { my ($opt, $_media) = @_; $run{chrooted_urpmi} = $opt; 1 }, "Activating chroot media" ], [ "", "clean-all", 0, "", "Clean all remaining chroots for all the users", sub { $run{clean_all} = 1 }, "Activating clean chroot flag" ], @@ -323,7 +324,7 @@ plog_init($program_name, $run{logfd} || $LOG, 7, 1); # For parsing command line # Display version information # -my $version = '0.7.8.1'; +my $version = '0.9.0'; plog("MSG", "This is iurt version $version"); my $todo = parseCommandLine($program_name, \@ARGV, \@params); @@ -380,6 +381,7 @@ my %config_usage = ( desc => 'List of packages needed for the chroot creation', default => [ 'basesystem-minimal', + 'makedev', 'rpm-build', 'sudo', 'urpmi', @@ -459,7 +461,7 @@ my %config_usage = ( default => '' }, max_command_retry => { - "Maximum number of retry Iurt will perform for a given command", + desc => 'Maximum number of retry Iurt will perform for a given command', default => 20 }, no_mail => { @@ -723,8 +725,6 @@ sub rebuild_one { my $retry = 0; retry: - $urpmi->clean_urpmi_process; - my ($srpm_name) = $srpm =~ /(?:.*:)?(.*)-[^-]+-[^-]+\.src\.rpm$/; $srpm_name or return $srpm; @@ -782,24 +782,25 @@ retry: mkdir $log_dir; -d $log_dir or die "FATAL: could not create $log_dir (check permissions and group ownerships)"; + # We may have media not used to create the chroot (when building for updates_testing + # or if using additional_media) so need to update basesystem packages. + $urpmi->update($chroot_tmp); + plog('INFO', "Install build dependencies for $srpm"); my $path_srpm = "$chroot_tmp/home/$luser/rpmbuild/SRPMS/"; - # on x86_64 the rpm database is getting corrupted and sometimes - # rpm do not found anymore installed packages, retrying several - # time to be sure something is really broken - my $ok = $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, 'install_deps', "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", { maintainer => $maintainer }, "$path_srpm/$srpm"); if (!$ok) { $run{status}{$srpm} ||= 'install_deps_failure'; return $srpm; } - # try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: - # No such file or directory - # system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null"); - # system("$sudo chroot $chroot_tmp rpm --rebuilddb &> /dev/null"); - + $ok = $urpmi->install_dynamic_buildrequires(\%run, $config, $chroot_tmp, $luser, $spec, $srpm); + if (!$ok) { + $run{status}{$srpm} ||= 'install_deps_failure'; + return $srpm; + } + perform_command("rpm --root $chroot_tmp -qa | sort", \%run, $config, logname => "rpm_qa", @@ -808,7 +809,8 @@ retry: debug_mail => $run{debug}, log => $log_dir); # or next; As this failed quite often, do not stop plog('NOTIFY', "Building $srpm"); - my $command = "rpmbuild --rebuild $run{with_flags} /home/$luser/rpmbuild/SRPMS/$srpm"; + my $target_arch = check_noarch("$path_srpm/$srpm") ? 'noarch' : $run{my_arch}; + my $command = "rpmbuild --target $target_arch --rebuild $run{with_flags} /home/$luser/rpmbuild/SRPMS/$srpm"; if ($run{stop}) { $urpmi->install_packages('chroot', $chroot_tmp, $local_spool, 'configure', "[ADMIN] installation of urpmi and sudo failed in the chroot $run{my_arch}", { check => 1, maintainer => $config->{admin} }, 'urpmi', 'sudo'); add_sudoers($chroot_tmp, $luser); @@ -865,9 +867,8 @@ retry: if ($config->{check_binary_file}) { $urpmi->install_packages($srpm, $chroot_tmp, $local_spool, 'binary_test', "[REBUILD] binaries packages generated from $srpm do not install correctly", { maintainer => $maintainer } ,@packages) or return $srpm; } else { - my $successfile = "$local_spool/log/$srpm/binary_test_$srpm-1.log"; - open my $f, ">$successfile"; - print $f "$srpm build ok"; + my $successfile = "$log_dir/binary_test_$srpm-1.log"; + output($successfile, "$srpm build ok"); } if ($run{debug}) { @@ -882,15 +883,19 @@ retry: } else { # drop packages and logs if we only want failure logs if ($run{delete_on_success}) { - system("rm -rf $local_spool/log/$srpm/"); + system("rm -rf $log_dir"); } elsif (!$run{discard_packages}) { plog('OK', "build successful, copying packages to $local_spool."); if (system("cp $chroot_tmp/home/$luser/rpmbuild/RPMS/*/*.rpm $local_spool &>/dev/null")) { # If copy fails (like disk full), report a failure and delete partially copied files plog('ERROR', "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpmbuild/RPMS/ to $local_spool ($!)"); foreach my $package (@packages) { + $package =~ s|.*/||; unlink "$local_spool/$package"; } + } else { + my $packagesfile = "$log_dir/packages.$run{my_arch}.$run{run}.log"; + output($packagesfile, sort(map { m|([^/]*)$|; "$1\n" } @packages)); } } @@ -1082,11 +1087,9 @@ sub check_pid { my $state = `ps h -o state $pid`; chomp $state; if ($time < time()-36000 || $state eq 'Z') { - my $msg = "an other iurt pid $pid is running for a very long time or is zombie, killing it"; - plog($msg); + plog("an other iurt pid $pid is running for a very long time or is zombie, killing it"); my $i; while ($i < 5 && getpgrp $pid != -1) { - set_alarm_message($msg); kill_for_good($pid); $i++; sleep 1; diff --git a/iurt_root_command b/iurt_root_command index 87b5c5e..e64b1f7 100755 --- a/iurt_root_command +++ b/iurt_root_command @@ -45,7 +45,7 @@ $run{todo} = []; # [ "", $program_name, 0, "[--verbose <level>] [--modprobe <module>] - [--mkdir [--parents] <dir1> <dir2> ... <dirn>]", + [--mkdir [--parents] [--mode <mode>] <dir1> <dir2> ... <dirn>]", "$program_name is a perl script to execute commands which need root privilege, it helps probram which needs occasional root privileges for some commands.", sub { $arg or usage($program_name, \@params) }, String::Escape::elide(join(' ', "Running $program_name", @ARGV), 120) ], @@ -77,7 +77,7 @@ $run{todo} = []; \&ln, "Linking files" ], [ "", "mkdir", [ - ["", "mkdir", -1, "[--parents] <dir1> <dir2> ... <dirn>", "mkdir create the given path", + ["", "mkdir", -1, "[--parents] [--mode <mode>] <dir1> <dir2> ... <dirn>", "mkdir create the given path", sub { my ($tmp, @arg) = @_; $tmp->[0] ||= {}; @@ -87,7 +87,10 @@ $run{todo} = []; ["p", "parents", 0, "", "Also create needed parents directories", sub { my ($tmp) = @_; $tmp->[0]{parents} = 1; 1 }, "Set the parents flag"], - ], "[--parents] <dir1> <dir2> ... <dirn>", + ["m", "mode", 1, "", + "Set the given mode on created directories", + sub { my ($tmp, $arg) = @_; $tmp->[0]{mode} = $arg; 1 }, "Set the mode flag"], + ], "[--parents] [--mode <mode>] <dir1> <dir2> ... <dirn>", "mkdir create the given path", \&mkdir, "Creating the path" ], @@ -244,6 +247,9 @@ sub mkdir { } else { mkdir $path; } + if ($opt->{mode}) { + chmod $opt->{mode}, $path; + } } 1; } diff --git a/lib/Iurt/Chroot.pm b/lib/Iurt/Chroot.pm index 1427aa3..8a2e04b 100644 --- a/lib/Iurt/Chroot.pm +++ b/lib/Iurt/Chroot.pm @@ -96,8 +96,12 @@ sub clean_and_build_chroot { } if ($run->{additional_media} && $run->{additional_media}{repository}) { - _setup_additional_media($run, $config, $chroot) or return; + if (!_setup_additional_media($run, $config, $chroot)) { + _clean_mounts($run, $config, $chroot); + return; + } } + 1; } @@ -232,7 +236,7 @@ sub clean_all_chroot_tmp { } foreach (readdir($dir)) { /$prefix/ or next; - clean_chroot($run, $config, "$chroot_dir/$_"); + clean_chroot("$chroot_dir/$_", $run, $config); } closedir $dir; } @@ -268,12 +272,13 @@ sub check_mounted { } sub check_chroot_need_update { - my ($tmp_chroot, $run) = @_; + my ($tmp_chroot, $run, $config) = @_; - my $tmp_urpmi = mktemp("$tmp_chroot/tmp.XXXXXX"); + sudo($config, '--mkdir', "-m", 01777, "$tmp_chroot/tmp"); + my $tmp_urpmi = mktemp("$tmp_chroot/tmp/tmp.XXXXXX"); mkdir_p("$tmp_urpmi/tmp"); my @installed_pkgs = grep { !/^gpg-pubkey/ } chomp_(cat_("$tmp_chroot/var/log/qa")); - my @available_pkgs = chomp_(`urpmq --urpmi-root $tmp_urpmi --use-distrib $run->{urpmi}{distrib_url} --list -f 2>/dev/null`); + my @available_pkgs = chomp_(`urpmq --urpmi-root $tmp_urpmi --use-distrib $run->{urpmi}{distrib_url} --list -f`); my @removed_pkgs = difference2(\@installed_pkgs, \@available_pkgs); rm_rf($tmp_urpmi); @@ -303,16 +308,6 @@ sub create_build_chroot { $ret = create_build_chroot_tar($chroot, $chroot_ref, $run, $config); } - if ($ret) { - my $urpmi = $run->{urpmi}; - if ($urpmi->{use__urpmi_root} && !$run->{chrooted_urpmi}) { - if (!$urpmi->add_media__urpmi_root($chroot, $config->{base_media})) { - plog('ERROR', "urpmi.addmedia --urpmi-root failed"); - return; - } - } - } - if ($ret && $use_netns) { sudo($config, '--netns_create', $chroot); } @@ -322,47 +317,44 @@ sub create_build_chroot { sub create_build_chroot_tar { my ($chroot, $chroot_tar, $run, $config) = @_; - my $tmp_chroot = mktemp("$chroot.tmp.XXXXXX"); my $rebuild; - my $clean = sub { - plog("Remove temporary chroot"); - sudo($config, '--rm', '-r', $tmp_chroot); - }; plog('NOTIFY', "creating chroot"); - mkdir_p($tmp_chroot); if (!-f $chroot_tar) { plog("rebuild chroot tarball"); $rebuild = 1; } elsif (!$run->{fixed_media}) { - plog('DEBUG', "decompressing /var/log/qa from $chroot_tar in $tmp_chroot"); - sudo($config, '--untar', $chroot_tar, $tmp_chroot, "./var/log/qa"); - $rebuild = check_chroot_need_update($tmp_chroot, $run); + my $tmp_chroot = mktemp("$chroot.tmp.XXXXXX"); + sudo($config, "--mkdir", "-p", "$tmp_chroot"); + plog('DEBUG', "decompressing /var/log/qa from $chroot_tar in $tmp_chroot"); + sudo($config, '--untar', $chroot_tar, $tmp_chroot, "./var/log/qa"); + $rebuild = check_chroot_need_update($tmp_chroot, $run, $config); + sudo($config, '--rm', '-r', $tmp_chroot); } + sudo($config, '--rm', '-r', $chroot); + # Create this directory as root as it will be / of the chroot + sudo($config, "--mkdir", "-p", "$chroot"); if ($rebuild) { - sudo($config, '--rm', '-r', $chroot); - if (!build_chroot($run, $config, $tmp_chroot)) { + if (!build_chroot($run, $config, $chroot)) { plog('NOTIFY', "creating chroot failed."); - $clean->(); + sudo($config, '--rm', '-r', $chroot); return; } - sudo($config, "--tar", $chroot_tar, $tmp_chroot); - # This rename may fail if for example tmp chroots are in another FS - # This does not matter as it will then be rm + untar - rename $tmp_chroot, $chroot; - } - - if (!-d $chroot) { - plog('DEBUG', "recreate chroot $chroot"); - plog('NOTIFY', "recreate chroot"); - mkdir_p $chroot; + plog('NOTIFY', "chroot recreated in $chroot"); + my $tmp_tar = mktemp("$chroot_tar.tmp.XXXXXX"); + sudo($config, "--tar", $chroot_tar, $chroot); + if (rename($tmp_tar, $chroot_tar)) { + plog('NOTIFY', "archive creation failed."); + unlink($tmp_tar); + return; + } + plog('NOTIFY', "chroot archived in $chroot_tar"); + } else { sudo($config, '--untar', $chroot_tar, $chroot); - plog('NOTIFY', "chroot recreated in $chroot_tar (live in $chroot)"); + plog('NOTIFY', "chroot recreated in $chroot (from $chroot_tar)"); } - - $clean->(); 1; } @@ -373,7 +365,7 @@ sub create_build_chroot_btrfs { plog('NOTIFY', "creating btrfs chroot"); # TODO: Handle $run{fixed_media} - if (check_chroot_need_update($chroot_ref, $run)) { + if (check_chroot_need_update($chroot_ref, $run, $config)) { sudo($config, '--btrfs_delete', $chroot_ref); if (!sudo($config, '--btrfs_create', $chroot_ref)) { plog('ERROR', "creating btrfs subvolume failed."); @@ -408,16 +400,13 @@ sub build_chroot { # install chroot my $urpmi = $run->{urpmi}; # perl_checker: $urpmi = Iurt::Urpmi->new - if ($urpmi->{use__urpmi_root}) { - if (!$urpmi->add_media__urpmi_root($tmp_chroot, $config->{base_media})) { - plog('ERROR', "urpmi.addmedia --urpmi-root failed"); - return 0; - } - $urpmi->set_command__urpmi_root($tmp_chroot); - } else { - $urpmi->set_command__use_distrib($tmp_chroot); + if (!$urpmi->add_media__urpmi_root($tmp_chroot, $config->{base_media})) { + plog('ERROR', "urpmi.addmedia --urpmi-root failed"); + return 0; } + $urpmi->set_command($tmp_chroot); + # (blino) install meta-task first for prefer.vendor.list to be used foreach my $packages ([ 'meta-task' ], $config->{basesystem_packages}) { if (!$urpmi->install_packages( diff --git a/lib/Iurt/Config.pm b/lib/Iurt/Config.pm index 32b87e4..3b8bbe1 100644 --- a/lib/Iurt/Config.pm +++ b/lib/Iurt/Config.pm @@ -1,7 +1,6 @@ package Iurt::Config; use base qw(Exporter); -use RPM4::Header; use Data::Dumper; use MDK::Common; use Iurt::Util qw(plog); @@ -17,8 +16,6 @@ our @EXPORT = qw( get_date get_prefix get_author_email - check_arch - check_noarch get_package_prefix get_mandatory_arch get_target_arch @@ -26,15 +23,13 @@ our @EXPORT = qw( ); our %arch_comp = ( - 'i586' => { 'i386' => 1, 'i486' => 1, 'i586' => 1 }, - 'i686' => { 'i386' => 1, 'i486' => 1, 'i586' => 1, 'i686' => 1 }, - 'x86_64' => { 'x86_64' => 1 }, - 'ppc' => { 'ppc' => 1 }, - 'ppc64' => { 'ppc' => 1, 'ppc64' => 1 }, + 'i586' => { 'i386' => 1, 'i486' => 1 }, + 'i686' => { 'i386' => 1, 'i486' => 1, 'i586' => 1 }, + 'ppc64' => { 'ppc' => 1 }, 'armv5tejl' => { 'armv5tl' => 1 }, 'armv5tel' => { 'armv5tl' => 1 }, - 'armv5tl' => { 'armv5tl' => 1 }, - 'armv7l' => { 'armv5tl' => 1, 'armv7hl' => 1, 'armv7hnl' => 1 }, + 'armv7l' => { 'armv5tl' => 1, 'armv7hnl' => 1 }, + 'armv8l' => { 'armv5tl' => 1, 'armv7hl' => 1, 'armv7hnl' => 1 }, ); @@ -151,48 +146,6 @@ sub get_author_email { return $authoremail; } -sub check_noarch { - my ($rpm) = @_; - my $hdr = RPM4::Header->new($rpm); - - # Stupid rpm doesn't return an empty list so we must check for (none) - - my ($build_archs) = $hdr->queryformat('%{BUILDARCHS}'); - - if ($build_archs ne '(none)') { - ($build_archs) = $hdr->queryformat('[%{BUILDARCHS} ]'); - my @list = split ' ', $build_archs; - return 1 if member('noarch', @list); - } - - return 0; -} - -sub check_arch { - my ($rpm, $arch) = @_; - my $hdr = RPM4::Header->new($rpm); - - # Stupid rpm doesn't return an empty list so we must check for (none) - - my ($exclusive_arch) = $hdr->queryformat('%{EXCLUSIVEARCH}'); - - if ($exclusive_arch ne '(none)') { - ($exclusive_arch) = $hdr->queryformat('[%{EXCLUSIVEARCH} ]'); - my @list = split ' ', $exclusive_arch; - return 0 unless member($arch, @list); - } - - my ($exclude_arch) = $hdr->queryformat('[%{EXCLUDEARCH} ]'); - - if ($exclude_arch ne '(none)') { - ($exclude_arch) = $hdr->queryformat('[%{EXCLUDEARCH} ]'); - my @list = split ' ', $exclude_arch; - return 0 if member($arch, @list); - } - - return 1; -} - sub get_mandatory_arch { my ($config, $target) = @_; find { ref($_) eq 'ARRAY' } $config->{mandatory_arch}, diff --git a/lib/Iurt/Emi.pm b/lib/Iurt/Emi.pm index 851307c..bf76a64 100644 --- a/lib/Iurt/Emi.pm +++ b/lib/Iurt/Emi.pm @@ -6,12 +6,14 @@ use Iurt::Config qw(get_author_email get_mandatory_arch get_target_arch); use Iurt::Mail qw(sendmail); use Iurt::Queue qw(check_if_all_archs_processed); use Iurt::Util qw(plog); +use MDK::Common::File qw(touch); use MDK::Common::Func qw(find); use MDK::Common::DataStructure qw(difference2); use strict; our @EXPORT = qw( find_prefixes_ready_to_upload + record_uploaded_packages upload_prefix_in_media ); @@ -150,11 +152,10 @@ sub upload_prefix_in_media { plog('OK', " uploading $rpm in $done/$path"); } - # This should not happen :( + # When a package is not marked as noarch but only has noarch subpackages, there will be nothing + # left to upload when it finishes building on non mandaory arch. return unless @packages; - $user ||= $config->{upload_user}; - my $command = generate_upload_command($prefix, $media, $target, $user, \@packages, $o_finish, "$done$path/$youri_file"); plog('DEBUG', "running $command"); if (!system($command)) { @@ -193,3 +194,22 @@ sub upload_prefix_in_media { } } } + +sub record_uploaded_packages { + my ($config, $pkg_tree, $prefix, $media) = @_; + my $done = "$config->{queue}/done"; + my $path = $pkg_tree->{$prefix}{media}{$media}{path}; + + my %arches; + foreach my $pkg (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) { + my ($arch) = $pkg =~ /\.([^.]*)\.rpm$/; + $arches{$arch} = 1 unless $arch eq 'src'; + } + # Only keep noarch if it's the only architecture + delete $arches{'noarch'} if 1 < keys %arches; + + foreach my $arch (keys %arches) { + touch("$done/$path/${prefix}_$arch.uploaded"); + } + touch("$done/$path/$prefix.upload"); +} diff --git a/lib/Iurt/Process.pm b/lib/Iurt/Process.pm index 46b8de2..c234d0b 100644 --- a/lib/Iurt/Process.pm +++ b/lib/Iurt/Process.pm @@ -14,8 +14,8 @@ our @EXPORT = qw( clean_process check_pid perform_command - set_alarm_message sudo + wait_for_lock ); my $sudo = '/usr/bin/sudo'; @@ -24,6 +24,7 @@ my $sudo = '/usr/bin/sudo'; Check that there is no other program running and create a pidfile lock I<$run> current running options +I<$exit> whether to exit when the lock is in use Return true. =cut @@ -32,7 +33,7 @@ Return true. # should be designed sub check_pid { - my ($run) = @_; + my ($run, $exit) = @_; my $pidfile = "$run->{pidfile_home}/$run->{pidfile}"; @@ -59,9 +60,7 @@ sub check_pid { if ($time < time()-7200 || $state eq 'Z') { my $i; - my $msg = "another instance [$pid] is too old, killing it"; - set_alarm_message($msg); - plog('WARN', $msg); + plog('WARN', "another instance [$pid] is too old, killing it"); while ($i < 5 && getpgrp $pid != -1) { kill_for_good($pid); @@ -69,9 +68,10 @@ sub check_pid { sleep 1; } } else { - plog('WARN', "another instance [$pid] is already running for ", - time()-$time, " seconds"); - exit(); + plog('WARN', "another instance [$pid] is already running for", + time()-$time, "seconds"); + exit() if $exit; + return; } } else { plog('WARN', "cleaning stale lockfile"); @@ -84,6 +84,20 @@ sub check_pid { $pidfile; } +sub wait_for_lock { + my ($run) = @_; + + plog('INFO', 'Waiting for ulri lock to be available'); + my $delay = 10; + my $pidfile = check_pid($run, 0); + while (!defined($pidfile)) { + plog('WARN', "waiting $delay seconds before retrying"); + sleep $delay; + $pidfile = check_pid($run, 0); + } + + $pidfile; +} sub fork_to_monitor { my ($run, $config, $logfile, %opt) = @_; @@ -105,18 +119,14 @@ sub fork_to_monitor { my (@stat) = stat $logfile; if ($stat[7] > $size_limit) { # FIXME: we left runaway processes (eg: urpmi) - my $msg = "Killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})"; - set_alarm_message($msg); - plog('ERROR', $msg); + plog('ERROR', "Killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})"); kill 14, "-$parent_pid"; exit(); } if ($opt{stalled_timeout} && $stat[9] && $stat[9] + $opt{stalled_timeout} < time()) { # If nothing was written to the logfile for more than stalled_timeout, check if the system seems busy if ((getload())[1] < 0.5) { - my $msg = "Killing current command because it seems blocked"; - set_alarm_message($msg); - plog('ERROR', $msg); + plog('ERROR', "Killing current command because it seems blocked"); kill 14, "-$parent_pid"; exit(); } @@ -125,8 +135,7 @@ sub fork_to_monitor { my $df = df $opt{log}; if ($df->{per} >= 99) { # FIXME: we left runaway processes (eg: urpmi) - set_alarm_message(my $msg = "Killing current command because running out of disk space at $opt{log} (only $df->{bavail}KB left)"); - plog('ERROR', $msg); + plog('ERROR', "Killing current command because running out of disk space at $opt{log} (only $df->{bavail}KB left)"); kill 14, "-$parent_pid"; exit(); } @@ -192,7 +201,7 @@ sub handle_wait_regexp { sub generate_comment { my ($run, $config, $output, $command, $comment, $pipe, $kill, %opt) = @_; if ($kill && $opt{type} ne 'shell') { - $comment = "Command killed after $opt{timeout}s: $command\n"; + $comment = "Command killed: $command\n"; my ($cmd_to_kill) = $command =~ /sudo(?: chroot \S+)? (.*)/; clean_process($cmd_to_kill); } elsif ($pipe) { @@ -210,13 +219,6 @@ sub generate_comment { } } -my $alarm_message; - -sub set_alarm_message { - my ($msg) = @_; - $alarm_message = $msg; -} - =head2 perform_command($command, $run, $config, %opt) Run a command and check various running parameters such as log size, timeout... @@ -258,19 +260,18 @@ sub perform_command { my $retry = $opt{retry} || 1; my $call_ret = 1; my ($err, $try); - my $logfile = "$opt{log}/$opt{logname}.$run->{run}.log"; + my $logfile = "$opt{log}/$opt{logname}.$run->{my_arch}.$run->{run}.log"; my $max_retry = max($config->{max_command_retry}, $retry); while ($retry) { $try++; - $logfile = "$opt{log}/$opt{logname}-$try.$run->{run}.log" if $opt{retry} > 1; + $logfile = "$opt{log}/$opt{logname}-$try.$run->{my_arch}.$run->{run}.log" if $opt{retry} > 1; my $pid = $opt{log} ? fork_to_monitor($run, $config, $logfile, %opt) : 0; eval { # handle timeout: - set_alarm_message("Timeout! (timeout was $opt{timeout})"); local $SIG{ALRM} = sub { - print $alarm_message, "\n"; + print "Killed! (probably because of the $opt{timeout} timeout)\n"; $kill = 1; die "alarm\n"; # NB: \n required }; @@ -306,7 +307,6 @@ sub perform_command { $err = 0 if any { $_ == $err } @{$opt{error_ok}}; # kill pid watching log file size - set_alarm_message("kill pid watching log file size"); kill_for_good($pid) if $pid; if ($perl_err) { # timed out @@ -382,7 +382,6 @@ sub kill_for_good { my ($pid) = @_; # try SIGALARM first: - set_alarm_message("Killing current command because it seems blocked"); kill 14, $pid; sleep 1; waitpid(-1, POSIX::WNOHANG); diff --git a/lib/Iurt/Queue.pm b/lib/Iurt/Queue.pm index 3930ca4..e78bd39 100644 --- a/lib/Iurt/Queue.pm +++ b/lib/Iurt/Queue.pm @@ -3,8 +3,9 @@ package Iurt::Queue; use base qw(Exporter); use File::Copy 'move'; use File::Path 'make_path'; +use File::stat 'stat'; use Iurt::Config qw(get_mandatory_arch get_target_arch); -use Iurt::File qw(read_line); +use Iurt::File qw(read_line create_file); use Iurt::Util qw(plog); use MDK::Common qw(cat_ find member partition); use strict; @@ -14,6 +15,10 @@ our @EXPORT = qw( cleanup_failed_build check_if_all_archs_processed check_if_mandatory_arch_failed + load_lock_file_data + record_bot_complete + remove_bot_from_package + schedule_next_retry ); sub apply_to_upload_tree { @@ -145,6 +150,51 @@ sub cleanup_failed_build { } } +sub load_lock_file_data { + my ($ent, $lock_path, $media, $config) = @_; + if ($lock_path !~ /\/(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.([\w-]+)\.(\d{14})\.(\d+)\.lock$/) { + plog('ERROR', "invalid lock file name: $lock_path"); + return; + } + my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6); + + $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; + plog('DEBUG', "found lock on $host/$arch for $prefix"); + + if ($arch =~ /noarch/) { + plog('DEBUG', "... and $prefix is noarch"); + $ent->{media}{$media}{arch}{noarch} = 1; + $arch =~ s/-.*//; + } + + $ent->{media}{$media}{arch}{$arch} = 1; + + my $time = read_line($lock_path); + $time = (split ' ', $time)[2]; + push @{$ent->{media}{$media}{bot}}, { + bot => $bot, + host => $host, + date => $date, + pid => $pid, + 'arch' => $arch, + 'time' => $time + }; +} + +sub remove_bot_from_package { + my ($ent, $media, $host, $pid) = @_; + @{$ent->{media}{$media}{bot}} = grep { $_->{host} ne $host || $_->{pid} != $pid} @{$ent->{media}{$media}{bot}}; +} + +sub record_bot_complete { + my ($run, $bot, $arch, $lock_file, $prefix, $ent, $media, $host, $pid) = @_; + plog('INFO', "delete lock file for $prefix on $host/$arch"); + unlink $lock_file; + $run->{bot}{$host}{$bot} = 0; + remove_bot_from_package($ent, $media, $host, $pid); + $ent->{media}{$media}{arch}{$arch} = 0; +} + sub get_upload_tree_state { our ($config) = @_; @@ -170,33 +220,13 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{srpm_name}{$name} = $srpm; } - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) { - my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6); + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.([\w-]+)\.(\d{14})\.(\d+)\.lock$/) { + my $prefix = $1; # Set path here too has we may have a lock without the src.rpm $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; - $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; - plog('DEBUG', "found lock on $host/$arch for $prefix"); - - if ($arch =~ /noarch/) { - plog('DEBUG', "... and $prefix is noarch"); - $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1; - $arch =~ s/-.*//; - } - - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; - - my $time = read_line("$todo/$f/$m/$s/$r"); - $time = (split ' ', $time)[2]; - push @{$pkg_tree{$prefix}{media}{$media}{bot}}, { - bot => $bot, - host => $host, - date => $date, - pid => $pid, - 'arch' => $arch, - 'time' => $time - }; + load_lock_file_data(\%{$pkg_tree{$prefix}}, "$todo/$f/$m/$s/$r", $media, $config); } if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) { @@ -206,6 +236,20 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{deps} = \@deps; } + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.retry$/) { + my $prefix = $1; + my $arch = $2; + my $mtime = stat("$todo/$f/$m/$s/$r")->mtime; + my $nb_failures = cat_("$todo/$f/$m/$s/$r"); + plog('DEBUG', "$prefix failed $nb_failures times, last one was as " . localtime($mtime)); + if ($mtime > time) { + plog('INFO', "Too early to retry $prefix, waiting until " . localtime($mtime)); + $pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1; + } else { + $pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures} = $nb_failures; + } + } } sub done_func { @@ -238,6 +282,8 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{media}{$media}{failed_arch}{$arch} = 1; } elsif ($result eq 'cancelled') { $pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch} = 1; + } elsif ($result eq 'uploaded') { + $pkg_tree{$prefix}{media}{$media}{uploaded_arch}{$arch} = 1; } else { plog('WARNING', "unknown state $arch.$result for $prefix"); } @@ -256,3 +302,21 @@ sub get_upload_tree_state { return %pkg_tree; } + +sub schedule_next_retry { + my ($config, $todo_dir, $prefix, $arch, $nb_failures) = @_; + + # If backoff_delays is not set, do nothing and retry forever + return 1 unless defined $config->{backoff_delays}; + + my $backoff_delays = $config->{backoff_delays}; + my $file = "$todo_dir/${prefix}_$arch.retry"; + create_file($file, $nb_failures+1); + if ($nb_failures >= scalar(@$backoff_delays)) { + plog('INFO', "$prefix failed too many times with a retriable error ($nb_failures)"); + return; + } + my $mtime = time + @$backoff_delays[$nb_failures]; + utime(time, $mtime, $file); + return 1; +} diff --git a/lib/Iurt/RPM.pm b/lib/Iurt/RPM.pm new file mode 100644 index 0000000..de09ba4 --- /dev/null +++ b/lib/Iurt/RPM.pm @@ -0,0 +1,55 @@ +package Iurt::RPM; + +use base qw(Exporter); +use RPM4::Header; +use MDK::Common; +use strict; + +our @EXPORT = qw( + check_arch + check_noarch +); + +sub check_noarch { + my ($rpm) = @_; + my $hdr = RPM4::Header->new($rpm); + + # Stupid rpm doesn't return an empty list so we must check for (none) + + my ($build_archs) = $hdr->queryformat('%{BUILDARCHS}'); + + if ($build_archs ne '(none)') { + ($build_archs) = $hdr->queryformat('[%{BUILDARCHS} ]'); + my @list = split ' ', $build_archs; + return 1 if member('noarch', @list); + } + + return 0; +} + +sub check_arch { + my ($rpm, $arch) = @_; + my $hdr = RPM4::Header->new($rpm); + + # Stupid rpm doesn't return an empty list so we must check for (none) + + my ($exclusive_arch) = $hdr->queryformat('%{EXCLUSIVEARCH}'); + + if ($exclusive_arch ne '(none)') { + ($exclusive_arch) = $hdr->queryformat('[%{EXCLUSIVEARCH} ]'); + my @list = split ' ', $exclusive_arch; + return 0 unless member($arch, @list); + } + + my ($exclude_arch) = $hdr->queryformat('[%{EXCLUDEARCH} ]'); + + if ($exclude_arch ne '(none)') { + ($exclude_arch) = $hdr->queryformat('[%{EXCLUDEARCH} ]'); + my @list = split ' ', $exclude_arch; + return 0 if member($arch, @list); + } + + return 1; +} + +1; diff --git a/lib/Iurt/Ulri.pm b/lib/Iurt/Ulri.pm index ce9d104..09be6b0 100755 --- a/lib/Iurt/Ulri.pm +++ b/lib/Iurt/Ulri.pm @@ -3,18 +3,155 @@ package Iurt::Ulri; use base qw(Exporter); use File::Path qw(make_path); use File::Temp qw(mktemp); -use Iurt::Config qw(get_author_email); +use Iurt::Config qw(config_init config_usage get_author_email); use Iurt::File qw(check_file_timeout); use Iurt::Mail qw(sendmail); -use Iurt::Util qw(plog ssh_setup ssh sput); +use Iurt::Util qw(plog ssh_setup sget ssh sput); use File::Slurp qw(read_file); +use MDK::Common qw(cat_); use strict; our @EXPORT = qw( build_package + fetch_logs_and_cleanup + load_config warn_about_failure ); +sub load_config { + my ($run) = @_; + my $HOME = $ENV{HOME}; + my $configfile = "$HOME/.upload.conf"; + my $sysconfigfile = "/etc/iurt/upload.conf"; + + my $config = {}; + foreach my $f ($configfile, $sysconfigfile) { + plog('DEBUG', "load config: $f"); + if (-f $f) { + $config = eval(cat_($f)) + or die "FATAL $run->{program_name}: syntax error in $f"; + last; + } + } + + my %config_usage = ( + admin => { + desc => 'mail address of the bot administrator', + default => 'distrib-admin@mandrivalinux.org' + }, + 'arch_translation' => { + desc => "Renaming of arch", + default => { 'sparc64' => 'sparcv9' } + }, + bot => { + desc => "List of bot able to compile the packages", + default => { + i586 => { + localhost => { + iurt => { + user => 'builder', + command => qq(iurt --copy_srpm --group --config local_spool /home/builder/iurt/__DIR__ --no_rsync --chrooted-urpmi -m __MEDIA__ -- http://localhost/distrib/ -p "__PACKAGER__" -r __TARGET__ __ARCH__), + packages => '/home/builder/iurt/', + }, + }, + }, + }, + }, + media => { + desc => 'Corresponding media to add given the current media', + default => { + default => { + "core/backports" => [ "core/backports", "core/release", "core/updates" ], + "core/backports_testing" => [ + "core/backports", "core/backports_testing", + "core/release", "core/updates" + ], + "core/release" => [ "core/release" ], + "core/updates" => [ "core/release", "core/updates" ], + "core/updates_testing" => [ + "core/release", "core/updates", "core/updates_testing" + ], + "nonfree/backports" => [ + "core/backports", "core/release", "core/updates", + "nonfree/backports", "nonfree/release", "nonfree/updates" + ], + "nonfree/backports_testing" => [ + "core/backports", "core/backports_testing", + "core/release", "core/updates", + "nonfree/backports", "nonfree/backports_testing", + "nonfree/release", "nonfree/updates" + ], + "nonfree/release" => [ "core/release", "nonfree/release" ], + "nonfree/updates" => [ + "core/release", "core/updates", + "nonfree/release", "nonfree/updates" + ], + "nonfree/updates_testing" => [ + "core/release", "core/updates", "core/updates_testing", + "nonfree/release", "nonfree/updates", "nonfree/updates_testing" + ], + "tainted/backports" => [ + "core/backports", "core/release", "core/updates", + "tainted/backports", "tainted/release", "tainted/updates" + ], + "tainted/backports_testing" => [ + "core/backports", "core/backports_testing", + "core/release", "core/updates", + "tainted/backports", "tainted/backports_testing", + "tainted/release", "tainted/updates" + ], + "tainted/release" => [ "core/release", "tainted/release" ], + "tainted/updates" => [ + "core/release", "core/updates", + "tainted/release", "tainted/updates" + ], + "tainted/updates_testing" => [ + "core/release", "core/updates", "core/updates_testing", + "tainted/release", "tainted/updates", "tainted/updates_testing" + ], + }, + }, + }, + faildelay => { + desc => "Time after which the rebuild is considered as a failure", + default => 36000 + }, + http_queue => { + desc => 'Address where log can be consulted', + default => 'http://pkgsubmit.mageia.org/uploads/' + }, + queue => { + desc => "Root of the tree where the packages to compile are located", + default => "$HOME/uploads" + }, + tmp => { + desc => "Temporary directory", + default => "$HOME/tmp" + }, + ssh_options => { + desc => "SSH options", + default => "-o ConnectTimeout=5 -o BatchMode=yes -o ServerAliveInterval=5" + }, + packager => { + desc => 'Default packager tag user by bot', + default => 'Mageia Team <http://www.mageia.org>' + }, + 'arch' => { + desc => 'Architectures list for each target', + default => { + default => [ 'i586', 'x86_64' ], + }, + }, + 'backoff_delays' => { + desc => 'List of delays in seconds before retrying retriable errors. Error becomes permanent after reaching the end of the list.', + default => [5*60, 30*60, 60*60, 120*60] + }, + ); + config_usage(\%config_usage, $config) if $run->{config_usage}; + config_init(\%config_usage, $config, $run); + return $config; +} + sub build_package { my ($config, $pkg_tree, $media, $prefix, $host, $arch, $bot) = @_; @@ -80,13 +217,13 @@ sub build_package { plog('DEBUG', "Will compile only with media $media_to_add"); $cmd =~ s!__MEDIA__!$media_to_add!g; - #- allow x86_64 hosts to build i586 packages - if ($arch eq 'i586') { - $cmd = "setarch i586 $cmd"; + #- force 32bit if needed, this allows to build 32 bits package on 64 bit hosts + if ($arch =~ /^(i.86|armv5tl|armv7hl)/) { + $cmd = "setarch linux32 $cmd"; } plog('DEBUG', "Build $pkgs"); - ssh($remote, "'echo PID=\$\$; exec $cmd $pkgs &>$prefix_dir/log/botcmd.\$(date +%s).\$(hostname -s).log' > $temp &"); + ssh($remote, "'echo PID=\$\$; exec $cmd $pkgs &>$prefix_dir/log/botcmd.\$(date +%s).$arch.\$(hostname -s).log' > $temp &"); # wait 10 seconds or until we have the log file # plus 20 seconds if it timeouts. @@ -144,6 +281,13 @@ sub get_pid_from_file { $pid; } +sub fetch_logs_and_cleanup { + my ($remote, $remote_dir, $target_dir) = @_; + make_path($target_dir); + sget($remote, "$remote_dir/log/*", $target_dir); + ssh($remote, "rm -rf $remote_dir"); +} + sub warn_about_failure { my ($config, $user, $ent, $arch, $fail_dir, $path, $prefix) = @_; my $text = join("\n", "Build of the following packages failed:\n", map { "- $_" } @{$ent->{srpms}}) . "\n"; @@ -161,11 +305,11 @@ sub warn_about_failure { $text .= "\nLog files generated:\n"; opendir my $DP1, "$fail_dir/$prefix/log/"; - foreach my $f1 (sort(readdir($DP1))) { + foreach my $f1 (readdir($DP1)) { next if ! -d "$fail_dir/$prefix/log/$f1" || $f1 =~ m/^\./; opendir my $DP2, "$fail_dir/$prefix/log/$f1"; - foreach my $f2 (readdir $DP2) { + foreach my $f2 (sort(readdir $DP2)) { next if $f2 =~ m/^\./; $text .= "$fpath/log/$f1/$f2\n"; } diff --git a/lib/Iurt/Urpmi.pm b/lib/Iurt/Urpmi.pm index 7ac6a74..53d0756 100644 --- a/lib/Iurt/Urpmi.pm +++ b/lib/Iurt/Urpmi.pm @@ -5,6 +5,7 @@ use RPM4::Header; use File::Basename; use File::NCopy qw(copy); use MDV::Distribconf::Build; +use MDK::Common qw(cat_); use Iurt::Chroot qw(add_local_user create_temp_chroot); use Iurt::Process qw(perform_command clean_process sudo); use Iurt::Config qw(get_package_prefix); @@ -21,31 +22,15 @@ sub new { my $config = $self->{config}; my $run = $self->{run}; - if ($run->{chrooted_urpmi}) { - $run->{chrooted_media} = $run->{chrooted_urpmi}{rooted_media} . - "/$run->{distro}/$run->{my_arch}"; - - # Now squash all slashes that don't follow colon - $run->{chrooted_media} =~ s|(?<!:)/+|/|g; - - plog('DEBUG', "installation media: $run->{chrooted_media}"); - } - $self->{use__urpmi_root} = !urpm::is_local_url($config->{repository}); $self->{distrib_url} = "$config->{repository}/$run->{distro}/$run->{my_arch}"; $self; } -sub set_command__urpmi_root { +sub set_command { my ($self, $chroot_tmp) = @_; - $self->{use_iurt_root_command} = 1; $self->{urpmi_command} = "urpmi $self->{urpmi_options} --urpmi-root $chroot_tmp"; } -sub set_command__use_distrib { - my ($self, $chroot_tmp) = @_; - $self->{use_iurt_root_command} = 1; - $self->{urpmi_command} = "urpmi $self->{urpmi_options} --use-distrib $self->{distrib_url} --root $chroot_tmp"; -} sub set_local_media { my ($self, $local_media) = @_; @@ -65,10 +50,7 @@ sub urpmi_command { my $run = $self->{run}; my $local_media = $self->{local_media}; - if ($run->{chrooted_urpmi}) { - # FIXME chrooted_urpmi should always use urpmi_root and not support local media - $self->{use__urpmi_root} ? &set_command__urpmi_root : &set_command__use_distrib; - + &set_command; # Here should be added only the needed media for the given package # main/release -> main/release # main/testing -> main/release main/testing @@ -82,18 +64,17 @@ sub urpmi_command { my $m_name = $m; $m_name =~ s,/,_,g; if (!add_media($self, $chroot_tmp, $m_name, - "$m_name $run->{chrooted_media}/media/$m")) { + "$m_name $self->{distrib_url}/media/$m")) { $run->{chrooted_urpmi} = 0; - plog('ERROR', "Failed to add media $m_name. Disabling chrooted_urpmi."); + plog('ERROR', "Failed to add media $m_name."); return; } } } else { # FIXME Do not hardcode Core - if (!add_media($self, $chroot_tmp, 'Core', "--distrib $run->{chrooted_media}")) { - if (!add_media($self, $chroot_tmp, 'Core', "--wget --distrib $run->{chrooted_media}")) { - $run->{chrooted_urpmi} = 0; - plog('ERROR', "Failed to add media $run->{chrooted_media}. Disabling chrooted_urpmi."); + if (!add_media($self, $chroot_tmp, 'Core', "--distrib $self->{distrib_url}")) { + if (!add_media($self, $chroot_tmp, 'Core', "--wget --distrib $self->{distrib_url}")) { + plog('ERROR', "Failed to add media $self->{distrib_url}."); return; } } @@ -149,9 +130,6 @@ sub urpmi_command { } return 1; - } else { - &set_command__use_distrib; - } } sub check_media_added { @@ -188,12 +166,9 @@ sub add_media { my $run = $self->{run}; my $config = $self->{config}; - plog("add chroot media: $run->{chrooted_media}"); + plog("add chroot media: $self->{distrib_url}"); - my $cmd = "chroot $chroot urpmi.addmedia $media"; - if ($run->{chrooted_urpmi}) { - $cmd = "urpmi-addmedia -v --urpmi-root $chroot $media"; - } + my $cmd = "urpmi-addmedia -v --urpmi-root $chroot $media"; perform_command($cmd, $run, $config, mail => $config->{admin}, @@ -208,6 +183,24 @@ sub add_media { 1; } +sub update { + my ($self, $chroot) = @_; + my $run = $self->{run}; + my $config = $self->{config}; + + plog('INFO', "updating packages in $chroot"); + + my $cmd = "urpmi --urpmi-root $chroot --auto-update --auto"; + perform_command($cmd, + $run, $config, + mail => $config->{admin}, + retry => 2, + use_iurt_root_command => 1, + debug_mail => $run->{debug}) + or return; + 1; +} + sub get_local_provides { my ($self) = @_; my $run = $self->{run}; @@ -341,8 +334,6 @@ sub _install_callback { # it seems the is needed urpmi error is due to something else (likely a # database corruption error). # my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g; - # - my @missing_deps = $output =~ /([^ \n]+) \(due to unsatisfied ([^[ \n]*)(?: ([^\n]*)|\[([^\n]*)\])?\)/g; # <mrl> 20071106 FIXME: This should not be needed anymore @@ -392,13 +383,13 @@ sub install_packages { @to_install or return 1; - (my $log_dirname = $title) =~ s/.*:(.*)\.src.rpm/$1/; + (my $log_dirname = $title) =~ s/^[^:]*:?([^:]*)\.(buildreqs\.nosrc|src).rpm/$1/; my $log_spool = "$local_spool/log/$log_dirname/"; mkdir $log_spool; - my @rpm = grep { !/\.src\.rpm$/ } @to_install; + my @rpm = grep { !/src\.rpm$/ } @to_install; if ($opt->{check} && -f "$chroot_tmp/bin/rpm" @@ -415,7 +406,7 @@ sub install_packages { if (!perform_command( join(' ', $self->{urpmi_command}, @options, @to_install), $run, $config, - use_iurt_root_command => $self->{use_iurt_root_command}, + use_iurt_root_command => 1, error => $error, logname => $log, hash => "${log}_$title", @@ -452,16 +443,16 @@ sub install_packages { $ok; } -sub clean_urpmi_process { - my ($self) = @_; - my $run = $self->{run}; - my $program_name = $run->{program_name}; - if (!$run->{chrooted_urpmi}) { - my $match = $self->{urpmi_command} or return; - if (!clean_process($match)) { - die "FATAL $program_name: Could not have urpmi working !"; - } +sub get_srpm_name { + my ($run, $config, $chroot_tmp, $srpm, $luser, $spec) = @_; + if (!perform_command(qq(chroot $chroot_tmp su $luser -c "rpmspec -q --qf %{NVR}.src.rpm --srpm /home/$luser/rpmbuild/SPECS/$spec > /home/$luser/rpmbuild/SPECS/$spec.srpm_name"), + $run, $config, + use_iurt_root_command => 1, + hash => "identify_$srpm")) { + plog("ERROR: failed to get the name of the generated src.rpm"); + return; } + return cat_("$chroot_tmp/home/$luser/rpmbuild/SPECS/$spec.srpm_name"); } # return ("exit_code", srpm, spec) @@ -518,7 +509,7 @@ sub recreate_srpm { my $filelist = `rpm -qlp $oldsrpm`; my ($name) = $srpm =~ /(?:.*:)?(.*)-[^-]+-[^-]+\.src\.rpm$/; foreach my $file (split "\n", $filelist) { - if ($file =~ /(.*)\.spec/) { + if ($file =~ /(.*)\.spec$/) { if (!$spec) { $spec = $file; } elsif ($1 eq $name) { @@ -545,8 +536,12 @@ sub recreate_srpm { # return 0 unless $ret; - my $file = (glob "$chroot_tmp/home/$luser/rpmbuild/SRPMS/$name-*.src.rpm")[0]; - my ($new_srpm) = basename($file); + my $new_srpm = get_srpm_name($run, $config, $chroot_tmp, $srpm, $luser, $spec); + if (!$new_srpm) { + plog("ERROR: failed to get the name of the generated src.rpm"); + return; + } + my $file = "$chroot_tmp/home/$luser/rpmbuild/SRPMS/$new_srpm"; my $prefix = get_package_prefix($srpm); my $newfile = "$chroot_tmp/home/$luser/rpmbuild/SRPMS/$prefix$new_srpm"; if (-f $file && $newfile ne $file) { @@ -561,4 +556,64 @@ sub recreate_srpm { ($ret, "$prefix$new_srpm", $spec); } +sub install_dynamic_buildrequires { + my ($self, $run, $config, $chroot_tmp, $luser, $spec, $srpm) = @_; + my $program_name = $run->{program_name}; + my $with_flags = $run->{with_flags}; + + plog('INFO', "handling dynamic buildrequires"); + + if (`rpm -qp --requires "$chroot_tmp/home/$luser/rpmbuild/SRPMS/$srpm"` !~ /rpmlib\(DynamicBuildRequires\)/) { + plog('DEBUG', "DynamicBuildRequires not required for $srpm"); + return 1; + } + + my $new_srpm = get_srpm_name($run, $config, $chroot_tmp, $srpm, $luser, $spec); + if (!$new_srpm) { + plog("ERROR: failed to get the name of the generated src.rpm"); + return; + } + my $nosrc = $new_srpm =~ s/src.rpm$/buildreqs.nosrc.rpm/r; + + while (1) { + # There is no way with perform_command to get the actual error code. + # We can however tell it that error 11 (missing BuildRequires) is ok so + # that we get an error if it fails for any other reason. + my $ret = perform_command(qq(chroot $chroot_tmp su $luser -c "rpmbuild -br $with_flags /home/$luser/rpmbuild/SPECS/$spec"), + $run, $config, + use_iurt_root_command => 1, + hash => "generatebuildrequires_$srpm" + ); + return 1 if $ret; + + # Unfortunately iurt_root_command hides the original error code so + # we need to find out if the file was created rather than relying on + # the command exiting with 11. + if (-f "$chroot_tmp/home/$luser/rpmbuild/SRPMS/$nosrc") { + plog("INFO: Some dynamic BuildRequires are missing"); + my @packages = ("$chroot_tmp/home/$luser/rpmbuild/SRPMS/$nosrc"); + if (!$self->install_packages( + $srpm, + $chroot_tmp, + $run->{local_spool}, + 'dynamic_buildrequires', + "[DYNAMIC BUILDREQUIRES] installation of dynamic BuildRequires failed for $srpm on $run->{my_arch}", + {}, + @packages + )) { + plog('ERROR', "Failed to install dynamic BuildRequires."); + return 0; + } + # TODO: Have a max retry counter for safety + # Delete the file if it exists so that we know if it got created again + unlink("$chroot_tmp/home/$luser/rpmbuild/SRPMS/$nosrc"); + } else { + # This was a failure for another reason, no point retrying + plog('ERROR', "Failed to generate dynamic BuildRequires."); + $run->{status}{$srpm} = 'build_failure'; + return 0; + } + } +} + 1; diff --git a/lib/Iurt/Util.pm b/lib/Iurt/Util.pm index f9dfa9f..bf726d0 100644 --- a/lib/Iurt/Util.pm +++ b/lib/Iurt/Util.pm @@ -167,7 +167,7 @@ execute I<@command>. Return the command output. sub sout { my $conf = shift; my ($opt, $user, $host) = @$conf; - `ssh $opt -x $user\@$host @_ 2>/dev/null`; + `ssh $opt -x $user\@$host "@_" 2>/dev/null`; } =item sget($handle, $from, $to) diff --git a/rebuild_perl_iurt b/rebuild_perl_iurt index 8bfed5c..c18da8e 100755 --- a/rebuild_perl_iurt +++ b/rebuild_perl_iurt @@ -23,33 +23,43 @@ if [ -z "$perlchroot" ]; then exit 1 fi -iurt --repository $repository --shell -r $distro $arch --use-old-chroot $perlchroot --chrooted-urpmi -m $media -- $repository <<"EOF" +iurt --repository $repository --shell -r $distro $arch --use-old-chroot $perlchroot --chrooted-urpmi -m $media -- $repository <<EOF set -x set -e -skip() { echo $*; } +skip() { echo \$*; } sudo urpmi.update -a sudo urpmi --auto mgarepo mkdir -p ~/.mgarepo -perl -pe 's/svn\+ssh:/svn:/' /etc/mgarepo.conf > ~/.mgarepo/config -perlapis_path=$HOME/perlapis.txt -[ -e $perlapis_path ] || rpm -q --provides perl-base | grep ^libperl.so. > $perlapis_path -for p in perl perl-Test-LeakTrace perl-List-MoreUtils-XS perl-List-MoreUtils perl-URPM perl-Locale-gettext; do mgarepo getsrpm -n $p; done +perl -pe 's/svn\\+ssh:/svn:/' /etc/mgarepo.conf > ~/.mgarepo/config +perlapis_path=\$HOME/perlapis.txt +[ -e \$perlapis_path ] || rpm -q --provides perl-base | grep ^libperl.so. > \$perlapis_path +for p in perl perl-Test-LeakTrace perl-PerlIO-utf8_strict perl-List-MoreUtils-XS perl-List-MoreUtils perl-URPM perl-Locale-gettext; do mgarepo getsrpm -n \$p; done rpm -ivh --nodeps *.src.rpm sudo urpmi --auto --no-verify-rpm rpmbuild/SPECS/*.spec -rpmbuild -ba rpmbuild/SPECS/perl.spec -sudo rpm -Uvh --nodeps --force $(ls rpmbuild/RPMS/*/perl-*.rpm | grep -v debuginfo) -rpmbuild -ba rpmbuild/SPECS/perl-Test-LeakTrace.spec +rpmbuild --target $arch -ba rpmbuild/SPECS/perl.spec +sudo rpm -Uvh --nodeps --force \$(ls rpmbuild/RPMS/*/perl-*.rpm | grep -v debuginfo) +rpmbuild --target $arch -ba rpmbuild/SPECS/perl-Test-LeakTrace.spec sudo rpm -Uvh --force rpmbuild/RPMS/*/perl-Test-LeakTrace*.rpm -rpmbuild -ba rpmbuild/SPECS/perl-List-MoreUtils-XS.spec +rpmbuild --target $arch -ba rpmbuild/SPECS/perl-List-MoreUtils-XS.spec sudo rpm -Uvh --force rpmbuild/RPMS/*/perl-List-MoreUtils-XS*.rpm -PERL5LIB=$(rpm -ql perl-List-MoreUtils-XS perl-List-MoreUtils | perl -ne 's!/List/MoreUtils.pm!! && print') LIST_MOREUTILS_PP=1 rpmbuild -ba rpmbuild/SPECS/perl-List-MoreUtils.spec +PERL5LIB=\$(rpm -ql perl-List-MoreUtils-XS perl-List-MoreUtils | perl -ne 's!/List/MoreUtils.pm!! && print') LIST_MOREUTILS_PP=1 rpmbuild --target $arch -ba rpmbuild/SPECS/perl-List-MoreUtils.spec sudo rpm -Uvh --force rpmbuild/RPMS/*/perl-List-MoreUtils-*.rpm -rpmbuild -ba rpmbuild/SPECS/perl-URPM.spec -rpmbuild -ba rpmbuild/SPECS/perl-Locale-gettext.spec +rpmbuild --target $arch -ba rpmbuild/SPECS/perl-URPM.spec +rpmbuild --target $arch -ba rpmbuild/SPECS/perl-Locale-gettext.spec sudo rpm -Uvh --force rpmbuild/RPMS/*/perl-URPM-*.rpm rpmbuild/RPMS/*/perl-Locale-gettext-*.rpm -updatepkgs() { spec=$1; toupdate=; while read line; do set -- $line; name=$1; path=$2; rpm -q $name --quiet && toupdate="$toupdate $path"; done < <(rpm -q --specfile $spec --qf '%{name} rpmbuild/RPMS/%{arch}/%{name}-%{version}-%{release}.%{arch}.rpm\n' | egrep -v -- '-(debuginfo|__restore__)-'); [ -n "$toupdate" ] && sudo rpm -Uvh --force $toupdate; } -rebuild() { p=$1; mgarepo getsrpm -n $p; rpm -ivh @*:$p-*.src.rpm; spec=rpmbuild/SPECS/$p.spec; sudo urpmi --auto --no-verify-rpm $spec; rpmbuild -ba $spec; updatepkgs $spec; } -pkgs=$(LC_ALL=C rpm -q --qf '%{SOURCERPM}\n' --whatrequires $(cat $perlapis_path) | perl -lne '/^(\S+)-[^-]+-[^-]+$/ and print $1' | uniq); for p in $pkgs; do rebuild $p; done +updatepkgs() { spec=\$1; toupdate=; while read line; do set -- \$line; name=\$1; path=\$2; rpm -q \$name --quiet && toupdate="\$toupdate \$path"; done < <(rpm -q --target $arch --specfile \$spec --qf '%{name} rpmbuild/RPMS/%{arch}/%{name}-%{version}-%{release}.%{arch}.rpm\n' | egrep -v -- '-(debuginfo|__restore__)-'); [ -n "\$toupdate" ] && sudo rpm -Uvh --force \$toupdate; } +rebuild() { p=\$1; mgarepo getsrpm -n \$p; rpm -ivh @*:\$p-*.src.rpm; spec=rpmbuild/SPECS/\$p.spec; sudo urpmi --auto --no-verify-rpm \$spec; rpmbuild --target $arch -ba \$spec; updatepkgs \$spec; } +pkgs=\$(LC_ALL=C rpm -q --qf '%{SOURCERPM}\n' --whatrequires \$(cat \$perlapis_path) | perl -lne '/^(\S+)-[^-]+-[^-]+$/ and print \$1' | uniq); for p in \$pkgs; do rebuild \$p; done EOF echo $perlchroot + +cat <<EOF +# ONCE THAT IS DONE, WE CAN REBUILD OTHER PERL PKGS USING THE REGULAR BS: +# ======================================================================== +# Use the following to rebuild binary packages _not_ depending on libperl.so: +# Those will install w/o dep issue but will fail at runtime with eg: +# lib/RPM2.c: loadable library and perl binaries are mismatched (got handshake key 0xcd00080, needed 0xed00080) +(for i in perl*;do rpm -qpl \$i|fgrep -q .so&&echo \$i;done>>/tmp/PERLBIN) +(for i in \$(</tmp/PERLBIN);do rpm -qpR \$i|fgrep -q libperl||rpm -qp --qf '%{sourcerpm}\n' \$i;done>>/tmp/PERLBIN2) +EOF diff --git a/t/queue.t b/t/queue.t new file mode 100644 index 0000000..b480e67 --- /dev/null +++ b/t/queue.t @@ -0,0 +1,107 @@ +use Iurt::Queue; + +use Test::More; + +use Cwd; +use File::stat 'stat'; +use MDK::Common qw(cat_); + +$config = {}; + +my $media = 'core/release'; +my $bot = 'iurt'; + +my $b1 = { + bot => $bot, + host => 'h1', + date => 20220401, + pid => 1, + 'arch' => 'a1', + 'time' => 42 +}; + +my $b2 = { + bot => $bot, + host => 'h1', + date => 20220401, + pid => 2, + 'arch' => 'a2', + 'time' => 42 +}; + +my $b3 = { + bot => $bot, + host => 'h2', + date => 20220401, + pid => 1, + 'arch' => 'a1', + 'time' => 42 +}; + +sub create_ent { + my %ent; + $ent->{media}{$media}{bot} = [$b1, $b2, $b3]; + return $ent; +} + +my $ent, $expected_ent; + +$ent = create_ent(); +remove_bot_from_package($ent, $media, 'h1', 42); +is_deeply $ent->{media}{$media}{bot}, [$b1, $b2, $b3] or diag explain $ent->{media}{$media}{bot}; + +$ent = create_ent(); +remove_bot_from_package($ent, $media, 'h1', 1); +is_deeply $ent->{media}{$media}{bot}, [$b2, $b3] or diag explain $ent->{media}{$media}{bot}; + +$ent = create_ent(); +remove_bot_from_package($ent, $media, 'h2', 1); +is_deeply $ent->{media}{$media}{bot}, [$b1, $b2] or diag explain $ent->{media}{$media}{bot}; + +$ent = create_ent(); +remove_bot_from_package($ent, $media, 'h1', 2); +is_deeply $ent->{media}{$media}{bot}, [$b1, $b3] or diag explain $ent->{media}{$media}{bot}; + + +chdir 't' if -d 't'; +mkdir "tmp"; +my $dir = Cwd::cwd() . "/tmp"; +my $retry_file = "${dir}/test_noarch.retry"; +unlink $retry_file; + +sub verify_retry_file { + my ($test, $present, $content, $mtime) = @_; + if ($present) { + ok(-f $retry_file, "$test - $retry_file should exist"); + } else { + ok(!-f $retry_file, "$test - $retry_file should not exist"); + return; + } + is(cat_($retry_file), $content, "$test - $retry_file should contain $content"); + if ($mtime) { + my $t = stat($retry_file)->mtime; + # Allow 5s difference if running on a very slow machine + ok($t > $mtime - 5); + ok($t < $mtime + 5); + } +} + +unlink $retry_file; +ok(schedule_next_retry({'backoff_delays' => [1000, 2000]}, $dir, 'test', 'noarch', 0), "schedule_next_retry - first failure is retried"); +verify_retry_file("schedule_next_retry - first failure is retried", 1, 1, time+1000); + +unlink $retry_file; +ok(schedule_next_retry({'backoff_delays' => [1000, 2000]}, $dir, 'test', 'noarch', 1), "schedule_next_retry - one retry left is retried"); +verify_retry_file("schedule_next_retry - one retry left is retried", 1, 2, time+2000); + +unlink $retry_file; +ok(!schedule_next_retry({'backoff_delays' => [120, 1000]}, $dir, 'test', 'noarch', 2), "schedule_next_retry - no retry left is failed"); + +unlink $retry_file; +ok(!schedule_next_retry({'backoff_delays' => []}, $dir, 'test', 'noarch', 0), "schedule_next_retry - no retry is failed"); + +unlink $retry_file; +ok(schedule_next_retry({}, $dir, 'test', 'noarch', 0), "schedule_next_retry - always retry is retried"); +verify_retry_file("schedule_next_retry - always retry is retried", 0); + +done_testing(); @@ -18,28 +18,17 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -# compare and rebuild packages on different architecture -# -# TODO -# -# - create a configuration file to handle the various iurt running -# - get the content of the rebuild dir -# - run as many iurt as machines are available and gather results -# - the scheduler just take the results, launch new rebuild, and quit -# - use perl ssh and try to workarround the non working timeout when the -# remote machine is stalled -# - use submitter as packager, not generic name -# use strict; use MDK::Common qw(any cat_ if_ find); -use Iurt::Config qw(config_usage get_date config_init get_author_email check_arch check_noarch get_target_arch get_mandatory_arch); +use Iurt::Config qw(get_date get_author_email get_target_arch get_mandatory_arch); use Iurt::File qw(create_file); -use Iurt::Process qw(check_pid); -use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state); use Iurt::Mail qw(sendmail); +use Iurt::Process qw(check_pid); +use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state load_lock_file_data record_bot_complete schedule_next_retry); +use Iurt::RPM qw(check_arch check_noarch); use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput); -use Iurt::Ulri qw(build_package warn_about_failure); +use Iurt::Ulri qw(build_package fetch_logs_and_cleanup load_config warn_about_failure); use File::Copy 'move'; use File::Path 'make_path'; use File::Temp 'mktemp'; @@ -58,131 +47,7 @@ if (!$ENV{ULRI_LOG_FILE} || !open($LOG, '>>', $ENV{ULRI_LOG_FILE})) { plog_init($program_name, $LOG, 7, 1); -my $HOME = $ENV{HOME}; -my $configfile = "$HOME/.upload.conf"; -my $sysconfigfile = "/etc/iurt/upload.conf"; - -my $config = {}; -foreach my $f ($configfile, $sysconfigfile) { - plog('DEBUG', "load config: $f"); - if (-f $f) { - $config = eval(cat_($f)) - or die "FATAL $program_name: syntax error in $f"; - last; - } -} - -my %config_usage = ( - admin => { - desc => 'mail address of the bot administrator', - default => 'distrib-admin@mandrivalinux.org' - }, - 'arch_translation' => { - desc => "Renaming of arch", - default => { 'sparc64' => 'sparcv9' } - }, - bot => { - desc => "List of bot able to compile the packages", - default => { - i586 => { - localhost => { - iurt => { - user => 'builder', - command => qq(iurt --copy_srpm --group --config local_spool /home/builder/iurt/__DIR__ --no_rsync --chrooted-urpmi -m __MEDIA__ -- http://localhost/distrib/ -p "__PACKAGER__" -r __TARGET__ __ARCH__), - packages => '/home/builder/iurt/', - } , - }, - }, - }, - }, - media => { - desc => 'Corresponding media to add given the current media', - default => { - default => { - "core/backports" => [ "core/backports", "core/release", "core/updates" ], - "core/backports_testing" => [ - "core/backports", "core/backports_testing", - "core/release", "core/updates" - ], - "core/release" => [ "core/release" ], - "core/updates" => [ "core/release", "core/updates" ], - "core/updates_testing" => [ - "core/release", "core/updates", "core/updates_testing" - ], - "nonfree/backports" => [ - "core/backports", "core/release", "core/updates", - "nonfree/backports", "nonfree/release", "nonfree/updates" - ], - "nonfree/backports_testing" => [ - "core/backports", "core/backports_testing", - "core/release", "core/updates", - "nonfree/backports", "nonfree/backports_testing", - "nonfree/release", "nonfree/updates" - ], - "nonfree/release" => [ "core/release", "nonfree/release" ], - "nonfree/updates" => [ - "core/release", "core/updates", - "nonfree/release", "nonfree/updates" - ], - "nonfree/updates_testing" => [ - "core/release", "core/updates", "core/updates_testing", - "nonfree/release", "nonfree/updates", "nonfree/updates_testing" - ], - "tainted/backports" => [ - "core/backports", "core/release", "core/updates", - "tainted/backports", "tainted/release", "tainted/updates" - ], - "tainted/backports_testing" => [ - "core/backports", "core/backports_testing", - "core/release", "core/updates", - "tainted/backports", "tainted/backports_testing", - "tainted/release", "tainted/updates" - ], - "tainted/release" => [ "core/release", "tainted/release" ], - "tainted/updates" => [ - "core/release", "core/updates", - "tainted/release", "tainted/updates" - ], - "tainted/updates_testing" => [ - "core/release", "core/updates", "core/updates_testing", - "tainted/release", "tainted/updates", "tainted/updates_testing" - ], - }, - }, - }, - faildelay => { - desc => "Time after which the rebuild is considered as a failure", - default => 36000 - }, - http_queue => { - desc => 'Address where log can be consulted', - default => 'http://kenobi.mandriva.com/queue ' - }, - queue => { - desc => "Root of the tree where the packages to compile are located", - default => "$HOME/uploads" - }, - tmp => { - desc => "Temporary directory", - default => "$HOME/tmp" - }, - ssh_options => { - desc => "SSH options", - default => "-o ConnectTimeout=5 -o BatchMode=yes -o ServerAliveInterval=5" - }, - packager => { - desc => 'Default packager tag user by bot', - default => 'Mageia Team <http://www.mageia.org>' - }, - 'arch' => { - desc => 'Architectures list for each target', - default => { - default => [ 'i586', 'x86_64' ], - }, - }, -); -config_usage(\%config_usage, $config) if $run{config_usage}; -config_init(\%config_usage, $config, \%run); +my $config = load_config(\%run); my %untranslated_arch; foreach my $k (keys %{$config->{arch_translation}}) { @@ -192,7 +57,7 @@ foreach my $k (keys %{$config->{arch_translation}}) { $run{pidfile_home} = $config->{tmp}; $run{pidfile} = $program_name; -my $pidfile = check_pid(\%run); +my $pidfile = check_pid(\%run, 1); my ($fulldate, $daydate) = get_date(); @@ -238,7 +103,6 @@ foreach my $prefix (keys %pkg_tree) { # TODO: Make this parallel plog('MSG', "check build bot results"); -my %later; my $something_finished; foreach my $prefix (keys %pkg_tree) { my $ent = $pkg_tree{$prefix}; @@ -299,6 +163,7 @@ foreach my $prefix (keys %pkg_tree) { # Everything is fine, build is continuing! # Kill it if that package had failed on a mandatory arch if (check_if_mandatory_arch_failed($media, $ent, $config)) { + plog('INFO', "A mandatory arch had failed, killing the build on $host/$arch"); ssh($remote, "kill -TERM $pid"); $pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch} = 1; create_file("$done_dir/${prefix}_$arch.cancelled", "$bot $host"); @@ -316,23 +181,16 @@ foreach my $prefix (keys %pkg_tree) { } else { plog('FAIL', "$bot died on $host/$arch (status $proc_state), removing lock"); } - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0; - } - - # Either we are done or we should kill the build - plog('INFO', "delete lock file for $prefix"); - unlink $lock_file; + fetch_logs_and_cleanup($remote, $prefix_dir, "$done_dir/$prefix"); + record_bot_complete(\%run, $bot, $arch, $lock_file, $prefix, $ent, $media, $host, $pid); - $run{bot}{$host}{$bot} = 0; - - if (!$status) { - # TODO: fetch/clean the logs next bot; } my $fail; my $later; + my $done; # Check if the build bot finished on the other side # @@ -342,25 +200,31 @@ foreach my $prefix (keys %pkg_tree) { plog('DEBUG', $res); if ($r eq 'install_deps_failure') { plog('FAIL', "install deps failure, rebuild later: $p"); - $later{$prefix} = 1; - $later = 1; - # TODO: fetch/clean the logs - } + if (schedule_next_retry($config, $todo_dir, $prefix, $arch, $pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures})) { + $later = 1; + $pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1; + } else { + plog('FAIL', "Too many retries due to install_deps_failure: $p"); + $fail = 1; + } + } if ($r ne 'ok') { plog('FAIL', "$r: $p"); $fail = 1; + } else { + plog('OK', "build complete: $p"); + $done = 1; } } - if (!$fail && !$later) { + if ($done) { my @list = split "\n", sout($remote, "ls $prefix_dir"); my $error; - my $done; my $arch_check = join '|', $arch, if_($untranslated_arch{$arch}, @{$untranslated_arch{$arch}}); plog('MSG', "checking for $arch_check arch"); foreach my $result (@list) { - $result =~ /\.(src|$arch_check|noarch)\.rpm$/ or next; + $result =~ /\.rpm$/ or next; # do not copy the initial src package $result =~ /^$prefix/ and next; @@ -368,10 +232,6 @@ foreach my $prefix (keys %pkg_tree) { my $result_file = "$done_dir/${prefix}_$result"; plog('OK', "build ok: $result"); - if ($result =~ /\.$arch_check\.rpm$/) { - $done = 1; - } - plog('DEBUG', "copy files to done"); make_path($done_dir); if (sget($remote, "$prefix_dir/$result", @@ -387,40 +247,32 @@ foreach my $prefix (keys %pkg_tree) { # Add the package to the list of built ones, in case we fail another arch and need to cleanup push @{$ent->{rpms}}, $result_file; } - next if $error; + next bot if $error; - if ($done) { - if (check_if_mandatory_arch_failed($media, $ent, $config)) { - # Discard this arch as another mandatory one failed - cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $media, $arch, $config); - } else { - create_file("$done_dir/${prefix}_$arch.done", "$bot $host"); - $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch} = 1; - make_path("$done_dir/$prefix"); - sget($remote, "$prefix_dir/log/*", "$done_dir/$prefix"); - $something_finished = 1; - } - # Either we already fetched the success logs, or don't care - # as this success was discarded due to another failure. + if (check_if_mandatory_arch_failed($media, $ent, $config)) { + # Discard this arch as another mandatory one failed + plog('INFO', "A mandatory arch had failed, discarding the successful build from $host/$arch"); + cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $media, $arch, $config); ssh($remote, "rm -rf $prefix_dir"); - next bot; + } else { + create_file("$done_dir/${prefix}_$arch.done", "$bot $host"); + $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch} = 1; + fetch_logs_and_cleanup($remote, $prefix_dir, "$done_dir/$prefix"); + $something_finished = 1; } } - make_path($fail_dir); + record_bot_complete(\%run, $bot, $arch, $lock_file, $prefix, $ent, $media, $host, $pid); + + # In case of success we have now fetched packages and logs and cleaned up the remote machine + next bot if $done; unless ($pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch}) { - mkdir("$fail_dir/$prefix"); - if (sget($remote, "$prefix_dir/*", "$fail_dir/$prefix")) { - plog('ERROR', "copying from $host:$prefix_dir/ " . - "to $fail_dir/ failed ($!)"); - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0; - } + make_path($fail_dir); + fetch_logs_and_cleanup($remote, $prefix_dir, "$fail_dir/$prefix"); + $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0; } - # clean the log on the compilation machine - ssh($remote, "rm -rf $prefix_dir"); - # We got the logs but want to retry so don't record a failure next bot if $later; @@ -453,8 +305,6 @@ my %to_compile; # crash or just lock ulri somehow foreach my $prefix (sort keys %pkg_tree) { - next if $later{$prefix}; - my $ent = $pkg_tree{$prefix}; my $ready = 1; @@ -521,9 +371,13 @@ foreach my $prefix (sort keys %pkg_tree) { # need to find a bot for each arch foreach my $arch (@$arch_list) { + next if $pkg_tree{$prefix}{media}{$media}{later}{$arch}; + # Skip this arch if the package is already building for it or if it # should not be built on this arch or it has already failed or # succeeded. + # Check again for noarch here, as we may ave triggered a noarch build on + # previous architecture and should not start another one. next if $pkg_tree{$prefix}{media}{$media}{arch}{$arch}; next if $pkg_tree{$prefix}{media}{$media}{arch}{noarch}; next if $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch}; @@ -561,6 +415,7 @@ foreach my $prefix (sort keys %pkg_tree) { "$lock_arch.$bot.$host.$fulldate.$pid.lock"; plog('DEBUG', "create lock $lock_file"); create_file($lock_file, "$program_name $$", time()); + load_lock_file_data(\%{$pkg_tree{$prefix}}, $lock_file, $media, $config); last hosts; } @@ -605,38 +460,3 @@ plog('INFO', "jobs in queue:", %to_compile ? unlink $pidfile; exec "emi" if $something_finished; - -__END__ - -# ulri ends here - -Discussion ----------- - -20060802 (Warly) - -* I prefer creating a separate scheduler, so that it can eventually call - other bots. -* bots should be able to take packages by themselves. -* Iurt will perform several checks, they have to be standard and usable - by the maintainer, the results must be put in a visible directory or path -* We can put packages either in a dir or to prefix all files with the date - and uploader. Having all files in a dir will make the listing simpler. - Prefixing the files could be problematic if we base the rpm name and - version parsing on the filename. -* ulri knows the prefix, he could ask iurt to put the packages in a dir - with the same prefix. - -20060806 (Warly) - -* All the packages are put in done, then the final youri is run to put them - in queue/ - -20061104 (claudio) - -* Instead if having configuration defaults for our environment and using - ulri with the defaults, it would be nicer to have minimalistic/generic - defaults and install a configuration file in kenobi -* Ulri's configuration file could be renamed to .ulri.conf instead of - .upload.conf. ==> ok, it's also used by emi - |