Limit retries in case of install_deps_failure

A new config option (backoff_delays) was added to ulri.

It contains a list of delays (in seconds) between retries and default to
[5*60, 30*60, 60*60, 120*60] (5m, 30m, 1h, 2h).

When reaching the end of the list, we fail permanently.
To never retry, set it to [].
To retry forever, set it to undef.

1 files changed, 35 insertions, 1 deletions

diff --git a/lib/Iurt/Queue.pm b/lib/Iurt/Queue.pm
index 585294e..d877046 100644
--- a/lib/Iurt/Queue.pm
+++ b/lib/Iurt/Queue.pm
@@ -3,8 +3,9 @@ package Iurt::Queue;
 use base qw(Exporter);
 use File::Copy 'move';
 use File::Path 'make_path';
+use File::stat 'stat';
 use Iurt::Config qw(get_mandatory_arch get_target_arch);
-use Iurt::File qw(read_line);
+use Iurt::File qw(read_line create_file);
 use Iurt::Util qw(plog);
 use MDK::Common qw(cat_ find member partition);
 use strict;
@@ -17,6 +18,7 @@ our @EXPORT = qw(
     load_lock_file_data
     record_bot_complete
     remove_bot_from_package
+    schedule_next_retry
 );
 
 sub apply_to_upload_tree {
@@ -234,6 +236,20 @@ sub get_upload_tree_state {
 
 	    $pkg_tree{$prefix}{deps} = \@deps;
 	}
+
+	if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.retry$/) {
+	    my $prefix = $1;
+	    my $arch = $2;
+	    my $mtime = stat("$todo/$f/$m/$s/$r")->mtime;
+	    my $nb_failures = cat_("$todo/$f/$m/$s/$r");
+	    plog('DEBUG', "$prefix failed $nb_failures times, last one was as " . localtime($mtime));
+	    if ($mtime < time) {
+		plog('INFO', "Too early to retry $prefix");
+		$pkg_tree{$prefix}{media}{$media}{later}{$arch} = 1;
+	    } else {
+		$pkg_tree{$prefix}{media}{$media}{retries}{arch}{nb_failures} = $nb_failures;
+	    }
+	}
     }
 
     sub done_func {
@@ -286,3 +302,21 @@ sub get_upload_tree_state {
 
     return %pkg_tree;
 }
+
+sub schedule_next_retry {
+	my ($config, $todo_dir, $prefix, $arch, $nb_failures) = @_;
+
+	# If backoff_delays is not set, do nothing and retry forever
+	return 1 unless defined $config->{backoff_delays};
+
+	my $backoff_delays = $config->{backoff_delays};
+	my $file = "$todo_dir/${prefix}_$arch.retry";
+	create_file($file, $nb_failures+1);
+	if ($nb_failures >= scalar(@$backoff_delays)) {
+	    plog('INFO', "$prefix failed too many times with a retriable error ($nb_failures)");
+	    return;
+        }
+	my $mtime = time + @$backoff_delays[$nb_failures];
+	utime(time, $mtime, $file);
+	return 1;
+}