diff options
Diffstat (limited to 'lib/Iurt/Queue.pm')
| -rw-r--r-- | lib/Iurt/Queue.pm | 112 |
1 files changed, 88 insertions, 24 deletions
diff --git a/lib/Iurt/Queue.pm b/lib/Iurt/Queue.pm index 3930ca4..e78bd39 100644 --- a/lib/Iurt/Queue.pm +++ b/lib/Iurt/Queue.pm @@ -3,8 +3,9 @@ package Iurt::Queue; use base qw(Exporter); use File::Copy 'move'; use File::Path 'make_path'; +use File::stat 'stat'; use Iurt::Config qw(get_mandatory_arch get_target_arch); -use Iurt::File qw(read_line); +use Iurt::File qw(read_line create_file); use Iurt::Util qw(plog); use MDK::Common qw(cat_ find member partition); use strict; @@ -14,6 +15,10 @@ our @EXPORT = qw( cleanup_failed_build check_if_all_archs_processed check_if_mandatory_arch_failed + load_lock_file_data + record_bot_complete + remove_bot_from_package + schedule_next_retry ); sub apply_to_upload_tree { @@ -145,6 +150,51 @@ sub cleanup_failed_build { } } +sub load_lock_file_data { + my ($ent, $lock_path, $media, $config) = @_; + if ($lock_path !~ /\/(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.([\w-]+)\.(\d{14})\.(\d+)\.lock$/) { + plog('ERROR', "invalid lock file name: $lock_path"); + return; + } + my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6); + + $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; + plog('DEBUG', "found lock on $host/$arch for $prefix"); + + if ($arch =~ /noarch/) { + plog('DEBUG', "... and $prefix is noarch"); + $ent->{media}{$media}{arch}{noarch} = 1; + $arch =~ s/-.*//; + } + + $ent->{media}{$media}{arch}{$arch} = 1; + + my $time = read_line($lock_path); + $time = (split ' ', $time)[2]; + push @{$ent->{media}{$media}{bot}}, { + bot => $bot, + host => $host, + date => $date, + pid => $pid, + 'arch' => $arch, + 'time' => $time + }; +} + +sub remove_bot_from_package { + my ($ent, $media, $host, $pid) = @_; + @{$ent->{media}{$media}{bot}} = grep { $_->{host} ne $host || $_->{pid} != $pid} @{$ent->{media}{$media}{bot}}; +} + +sub record_bot_complete { + my ($run, $bot, $arch, $lock_file, $prefix, $ent, $media, $host, $pid) = @_; + plog('INFO', "delete lock file for $prefix on $host/$arch"); + unlink $lock_file; + $run->{bot}{$host}{$bot} = 0; + remove_bot_from_package($ent, $media, $host, $pid); + $ent->{media}{$media}{arch}{$arch} = 0; +} + sub get_upload_tree_state { our ($config) = @_; @@ -170,33 +220,13 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{srpm_name}{$name} = $srpm; } - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) { - my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6); + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.([\w-]+)\.(\d{14})\.(\d+)\.lock$/) { + my $prefix = $1; # Set path here too has we may have a lock without the src.rpm $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; - $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; - plog('DEBUG', "found lock on $host/$arch for $prefix"); - - if ($arch =~ /noarch/) { - plog('DEBUG', "... and $prefix is noarch"); - $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1; - $arch =~ s/-.*//; - } - - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; - - my $time = read_line("$todo/$f/$m/$s/$r"); - $time = (split ' ', $time)[2]; - push @{$pkg_tree{$prefix}{media}{$media}{bot}}, { - bot => $bot, - host => $host, - date => $date, - pid => $pid, - 'arch' => $arch, - 'time' => $time - }; + load_lock_file_data(\%{$pkg_tree{$prefix}}, "$todo/$f/$m/$s/$r", $media, $config); } if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) { @@ -206,6 +236,20 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{deps} = \@deps; } + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.retry$/) { + my $prefix = $1; + my $arch = $2; + my $mtime = stat("$todo/$f/$m/$s/$r")->mtime; + my $nb_failures = cat_("$todo/$f/$m/$s/$r"); + plog('DEBUG', "$prefix failed $nb_failures times, last one was as " . localtime($mtime)); + if ($mtime > time) { + plog('INFO', "Too early to retry $prefix, waiting until " . localtime($mtime)); + $pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1; + } else { + $pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures} = $nb_failures; + } + } } sub done_func { @@ -238,6 +282,8 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{media}{$media}{failed_arch}{$arch} = 1; } elsif ($result eq 'cancelled') { $pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch} = 1; + } elsif ($result eq 'uploaded') { + $pkg_tree{$prefix}{media}{$media}{uploaded_arch}{$arch} = 1; } else { plog('WARNING', "unknown state $arch.$result for $prefix"); } @@ -256,3 +302,21 @@ sub get_upload_tree_state { return %pkg_tree; } + +sub schedule_next_retry { + my ($config, $todo_dir, $prefix, $arch, $nb_failures) = @_; + + # If backoff_delays is not set, do nothing and retry forever + return 1 unless defined $config->{backoff_delays}; + + my $backoff_delays = $config->{backoff_delays}; + my $file = "$todo_dir/${prefix}_$arch.retry"; + create_file($file, $nb_failures+1); + if ($nb_failures >= scalar(@$backoff_delays)) { + plog('INFO', "$prefix failed too many times with a retriable error ($nb_failures)"); + return; + } + my $mtime = time + @$backoff_delays[$nb_failures]; + utime(time, $mtime, $file); + return 1; +} |