diff options
author | Pascal Terjan <pterjan@mageia.org> | 2023-12-10 21:43:07 +0000 |
---|---|---|
committer | Pascal Terjan <pterjan@mageia.org> | 2023-12-11 23:45:04 +0000 |
commit | c53d21cf38263206af292b48d6369ee1662bd4be (patch) | |
tree | b402732dc7c0fe1268a6873791b8fc180508c202 /lib | |
parent | 42466ac7d8bcd3fd1d1097d3dbf1f77cd1a2daa0 (diff) | |
download | iurt-c53d21cf38263206af292b48d6369ee1662bd4be.tar iurt-c53d21cf38263206af292b48d6369ee1662bd4be.tar.gz iurt-c53d21cf38263206af292b48d6369ee1662bd4be.tar.bz2 iurt-c53d21cf38263206af292b48d6369ee1662bd4be.tar.xz iurt-c53d21cf38263206af292b48d6369ee1662bd4be.zip |
Limit retries in case of install_deps_failure
A new config option (backoff_delays) was added to ulri.
It contains a list of delays (in seconds) between retries and default to
[5*60, 30*60, 60*60, 120*60] (5m, 30m, 1h, 2h).
When reaching the end of the list, we fail permanently.
To never retry, set it to [].
To retry forever, set it to undef.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/Iurt/Queue.pm | 36 |
1 files changed, 35 insertions, 1 deletions
diff --git a/lib/Iurt/Queue.pm b/lib/Iurt/Queue.pm index 585294e..d877046 100644 --- a/lib/Iurt/Queue.pm +++ b/lib/Iurt/Queue.pm @@ -3,8 +3,9 @@ package Iurt::Queue; use base qw(Exporter); use File::Copy 'move'; use File::Path 'make_path'; +use File::stat 'stat'; use Iurt::Config qw(get_mandatory_arch get_target_arch); -use Iurt::File qw(read_line); +use Iurt::File qw(read_line create_file); use Iurt::Util qw(plog); use MDK::Common qw(cat_ find member partition); use strict; @@ -17,6 +18,7 @@ our @EXPORT = qw( load_lock_file_data record_bot_complete remove_bot_from_package + schedule_next_retry ); sub apply_to_upload_tree { @@ -234,6 +236,20 @@ sub get_upload_tree_state { $pkg_tree{$prefix}{deps} = \@deps; } + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.retry$/) { + my $prefix = $1; + my $arch = $2; + my $mtime = stat("$todo/$f/$m/$s/$r")->mtime; + my $nb_failures = cat_("$todo/$f/$m/$s/$r"); + plog('DEBUG', "$prefix failed $nb_failures times, last one was as " . localtime($mtime)); + if ($mtime < time) { + plog('INFO', "Too early to retry $prefix"); + $pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1; + } else { + $pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures} = $nb_failures; + } + } } sub done_func { @@ -286,3 +302,21 @@ sub get_upload_tree_state { return %pkg_tree; } + +sub schedule_next_retry { + my ($config, $todo_dir, $prefix, $arch, $nb_failures) = @_; + + # If backoff_delays is not set, do nothing and retry forever + return 1 unless defined $config->{backoff_delays}; + + my $backoff_delays = $config->{backoff_delays}; + my $file = "$todo_dir/${prefix}_$arch.retry"; + create_file($file, $nb_failures+1); + if ($nb_failures >= scalar(@$backoff_delays)) { + plog('INFO', "$prefix failed too many times with a retriable error ($nb_failures)"); + return; + } + my $mtime = time + @$backoff_delays[$nb_failures]; + utime(time, $mtime, $file); + return 1; +} |