Limit retries in case of install_deps_failure

A new config option (backoff_delays) was added to ulri.

It contains a list of delays (in seconds) between retries and default to
[5*60, 30*60, 60*60, 120*60] (5m, 30m, 1h, 2h).

When reaching the end of the list, we fail permanently.
To never retry, set it to [].
To retry forever, set it to undef.

1 files changed, 14 insertions, 6 deletions

diff --git a/ulri b/ulri
index b814708..553fbd9 100755
--- a/ulri
+++ b/ulri
@@ -25,7 +25,7 @@ use Iurt::Config qw(config_usage get_date config_init get_author_email get_targe
 use Iurt::File qw(create_file);
 use Iurt::Mail qw(sendmail);
 use Iurt::Process qw(check_pid);
-use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state load_lock_file_data record_bot_complete);
+use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state load_lock_file_data record_bot_complete schedule_next_retry);
 use Iurt::RPM qw(check_arch check_noarch);
 use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput);
 use Iurt::Ulri qw(build_package fetch_logs_and_cleanup warn_about_failure);
@@ -169,6 +169,10 @@ my %config_usage = (
 	    default => [ 'i586', 'x86_64' ],
 	},
     },
+    'backoff_delays' => {
+	desc => 'List of delays in seconds before retrying retriable errors. Error becomes permanent after reaching the end of the list.',
+	default => [5*60, 30*60, 60*60, 120*60]
+    },
 );
 config_usage(\%config_usage, $config) if $run{config_usage};
 config_init(\%config_usage, $config, \%run);
@@ -227,7 +231,6 @@ foreach my $prefix (keys %pkg_tree) {
 # TODO: Make this parallel
 plog('MSG', "check build bot results");
 
-my %later;
 my $something_finished;
 foreach my $prefix (keys %pkg_tree) {
     my $ent = $pkg_tree{$prefix};
@@ -324,8 +327,13 @@ foreach my $prefix (keys %pkg_tree) {
 		plog('DEBUG', $res);
 		if ($r eq 'install_deps_failure') {
 		    plog('FAIL', "install deps failure, rebuild later: $p");
-		    $later{$prefix} = 1;
-		    $later = 1;
+		    if (schedule_next_retry($config, $todo_dir, $prefix, $arch, $pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures})) {
+			$later = 1;
+			$pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1;
+		    } else {
+			plog('FAIL', "Too many retries due to install_deps_failure: $p");
+			$fail = 1;
+		    }
 	        }
 		if ($r ne 'ok') {
 		    plog('FAIL', "$r: $p");
@@ -423,8 +431,6 @@ my %to_compile;
 # crash or just lock ulri somehow
 
 foreach my $prefix (sort keys %pkg_tree) {
-    next if $later{$prefix};
-
     my $ent = $pkg_tree{$prefix};
 
     my $ready = 1;
@@ -491,6 +497,8 @@ foreach my $prefix (sort keys %pkg_tree) {
 
 	# need to find a bot for each arch
 	foreach my $arch (@$arch_list) {
+	    next if $pkg_tree{$prefix}{media}{$media}{later}{$arch};
+
 	    # Skip this arch if the package is already building for it or if it
 	    # should not be built on this arch or it has already failed or
 	    # succeeded.