- urpmi, gurpmi

  o allow "retry" on aria2 download failure

(backported from trunk: r248522 r248523 r248538 r248541 r248554)

5 files changed, 58 insertions, 25 deletions

diff --git a/NEWS b/NEWS
index ddbeb440..06baf204 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,5 @@
+- urpmi, gurpmi
+  o allow "retry" on aria2 download failure
 - urpmi
   o fix --auto-update ignoring --media and default-media (#45097)
 - urpmi.addmedia
diff --git a/urpm/download.pm b/urpm/download.pm
index 9abe039d..d7387645 100644
--- a/urpm/download.pm
+++ b/urpm/download.pm
@@ -260,12 +260,17 @@ sub set_proxy {
     @res;
 }
 
-sub _error {
+sub _error_msg {
     my ($name) = @_;
 
     my $msg = $? & 127 ? N("%s failed: exited with signal %d", $name, $? & 127) :
                          N("%s failed: exited with %d", $name, $? >> 8);
-    die "$msg\n";
+    "$msg\n";
+}
+
+sub _error {
+    my ($name) = @_;
+    die _error_msg($name);
 }
 
 sub hide_password {
@@ -641,11 +646,10 @@ sub sync_prozilla {
 }
 
 sub sync_aria2 {
-    my ($options, @urls) = @_;
+    my ($urpm, $medium, $rel_files, $options) = @_;
 
     -x "/usr/bin/aria2c" or die N("aria2 is missing\n");
 
-    $options = { dir => $options } if !ref $options;
     #- force download to be done in cachedir to avoid polluting cwd.
     (my $cwd) = getcwd() =~ /(.*)/;
     chdir $options->{dir};
@@ -657,6 +661,7 @@ sub sync_aria2 {
 	"--auto-file-renaming=false",
 	'--ftp-pasv',
 	"--follow-metalink=mem",
+      $medium->{mirrorlist} ? (
 	'--metalink-enable-unique-protocol=false', # so that it can try both ftp and http access on the same server. aria2 will only do this on first calls
 	'--max-tries=1', # nb: not using $options->{retry}
 	'--lowest-speed-limit=20K', "--timeout", 3,
@@ -664,20 +669,36 @@ sub sync_aria2 {
         '--uri-selector=adaptive', "--server-stat-if=$stat_file", "--server-stat-of=$stat_file",
         $options->{is_versioned} ? () : '--max-file-not-found=3', # number of not found errors on different servers before aborting file download
         '--connect-timeout=6', # $CONNECT_TIMEOUT,
+      ) : (),
 	"-Z", "-j1",
 	($options->{'limit-rate'} ? "--max-download-limit=" . $options->{'limit-rate'} : ()),
 	($options->{resume} ? "--continue" : "--allow-overwrite=true"),
 	($options->{proxy} ? set_proxy({ type => "aria2", proxy => $options->{proxy} }) : ()),
 	(defined $options->{'aria2-options'} ? split /\s+/, $options->{'aria2-options'} : ()),
-	@urls);
-
-    my @urls_text = $options->{metalink} ? @{$options->{urls_text}} : @urls;
+        _create_metalink_($urpm, $medium, $rel_files, $options));
 
     $options->{debug} and $options->{debug}($aria2c_command);
 
     local $ENV{LC_ALL} = 'C';
     my $aria2_pid = open(my $aria2, "$aria2c_command |");
 
+    _parse_aria2_output($options, $aria2, $aria2_pid, $medium, $rel_files);
+
+    chdir $cwd;
+    if (!close $aria2) {
+	my $raw_msg = _error_msg('aria2');
+	my $msg = N("Failed to download %s", $rel_files->[0]);
+	if ($options->{ask_retry} && $options->{ask_retry}($raw_msg, $msg)) {
+	    $options->{is_retry} = 1;
+	    goto &sync_aria2;
+	}
+	die $raw_msg;
+    }
+}
+
+sub _parse_aria2_output {
+    my ($options, $aria2, $aria2_pid, $medium, $rel_files) = @_;
+
     my ($buf, $_total, $file) = ('', undef, undef);
 
     local $/ = \1; #- read input by only one char, this is slow but very nice (and it works!).
@@ -687,9 +708,12 @@ sub sync_aria2 {
 	if ($_ eq "\r" || $_ eq "\n") {
 	    $options->{debug}("aria2c: $buf") if $options->{debug};
 		if ($options->{callback}) {
-			if (!defined($file) && @urls_text) {
-				$file = shift @urls_text;
-				propagate_sync_callback($options, 'start', $file);
+			if (!defined($file) && @$rel_files) {
+				$file = $medium->{mirrorlist} ? 
+				  $medium->{mirrorlist} . ': ' . $medium->{'with-dir'} . "/$rel_files->[0]" :
+				  "$medium->{url}/$rel_files->[0]";
+				propagate_sync_callback($options, 'start', $file)
+				  if !delete $options->{is_retry};
 			}
     		    if ($buf =~ m!^\[#\d*\s+\S+:([\d\.]+\w*).([\d\.]+\w*)\S([\d]+)\S+\s+\S+\s*([\d\.]+)\s\w*:([\d\.]+\w*/\w)\s\w*:(\d+\w*)\]$!) {
 			    my ($total, $percent, $speed, $eta) = ($2, $3, $5, $6);
@@ -702,7 +726,8 @@ sub sync_aria2 {
 		    }
 		    if ($buf =~ m!Download\scomplete:\s\./!) {
 			propagate_sync_callback($options, 'end', $file);
-			$file = undef;
+			shift @$rel_files;
+			$file = undef;			
 		    } elsif ($buf =~ /ERR\|(.*)/) {
 			propagate_sync_callback($options, 'error', $file, $1);
 		    }
@@ -714,8 +739,6 @@ sub sync_aria2 {
 	    $buf .= $_;
 	}
     }
-    chdir $cwd;
-    close $aria2 or _error('aria2');
 }
 
 sub start_ssh_master {
@@ -868,7 +891,7 @@ sub sync_rel_to {
 #- deprecated, use sync_url() or sync_rel() instead
 #-
 #- $medium can be undef
-#- known options: quiet, resume, callback
+#- known options: quiet, resume, callback, ask_retry
 sub sync {
     my ($urpm, $medium, $files, %options) = @_;
 
@@ -908,13 +931,11 @@ sub _sync_webfetch_raw {
     } elsif (member($proto, 'ftp', 'http', 'https') || $options->{metalink}) {
 
 	my $preferred = preferred_downloader($urpm, $medium, \$options->{metalink});
-
-	my $sync = $urpm::download::{"sync_$preferred"} or die N("no webfetch found, supported webfetch are: %s\n", join(", ", urpm::download::ftp_http_downloaders()));
-
-	if ($options->{metalink}) {
-	    $options->{urls_text} = [ map { $medium->{mirrorlist} . ': ' . $medium->{'with-dir'} . "/$_" } @$rel_files ];
-	    $sync->($options, _create_metalink_($urpm, $medium, $rel_files, $options));
+	if ($preferred eq 'aria2') {
+	    sync_aria2($urpm, $medium, $rel_files, $options);
 	} else {
+	  my $sync = $urpm::download::{"sync_$preferred"} or die N("no webfetch found, supported webfetch are: %s\n", join(", ", urpm::download::ftp_http_downloaders()));
+
 	  my @l = @$files;
 	  while (@l) {
 	    my $half_MAX_ARG = 131072 / 2;
@@ -968,11 +989,11 @@ sub _create_metalink_ {
 
     # only use the 8 best mirrors, then we let aria2 choose
     require urpm::mirrors;
-    my @mirrors = map {
+    my @mirrors = $medium->{mirrorlist} ? (map {
 	# aria2 doesn't handle rsync
 	my @l = grep { urpm::protocol_from_url($_->{url}) ne 'rsync' } @$_;
 	_take_n_elem(8, @l);
-    } urpm::mirrors::list_urls($urpm, $medium, '');
+    } urpm::mirrors::list_urls($urpm, $medium, '')) : { url => $medium->{url} };
     
     my $metalinkfile = "$urpm->{cachedir}/$options->{media}.metalink";
     # Even if not required by metalink spec, this line is needed at top of
diff --git a/urpm/get_pkgs.pm b/urpm/get_pkgs.pm
index e5a9b5b8..9fb81b83 100644
--- a/urpm/get_pkgs.pm
+++ b/urpm/get_pkgs.pm
@@ -214,7 +214,9 @@ sub _download_packages_of_distant_media {
     if (urpm::download::sync_rel($urpm, $blist->{medium}, [ urpm::blist_to_filenames($blist) ],
 			     dir => "$cachedir/partial", quiet => $options{quiet}, 
 			     is_versioned => 1,
-			     resume => $urpm->{options}{resume}, callback => $options{callback})) {
+			     resume => $urpm->{options}{resume}, 
+			     ask_retry => $options{ask_retry},
+			     callback => $options{callback})) {
 	$urpm->{log}(N("...retrieving done"));
     } else {
 	$urpm->{error}(N("...retrieving failed: %s", $@));
diff --git a/urpm/main_loop.pm b/urpm/main_loop.pm
index 2e0fb70f..9583ed2d 100644
--- a/urpm/main_loop.pm
+++ b/urpm/main_loop.pm
@@ -101,6 +101,13 @@ foreach my $set (@{$state->{transaction} || []}) {
 	\@error_sources,
 	quiet => $options{verbose} < 0,
 	callback => $callbacks->{trans_log},
+        ask_retry => $callbacks->{ask_retry} || sub {
+	    my ($raw_msg, $msg) = @_;
+	    if (my $download_errors = delete $urpm->{download_errors}) {
+		$raw_msg = join("\n", @$download_errors, '');
+	    }
+	    $callbacks->{ask_yes_or_no}('', $raw_msg . "\n" . $msg . "\n" . N("Retry?"));
+	},
     );
     if (@error_sources) {
 	$_->[0] = urpm::download::hide_password($_->[0]) foreach @error_sources;
diff --git a/urpm/media.pm b/urpm/media.pm
index 67f4cefa..956dfb3b 100644
--- a/urpm/media.pm
+++ b/urpm/media.pm
@@ -1445,7 +1445,7 @@ sub _download_MD5SUM_and_check {
     undef;
 }
 
-#- options: callback, force, nomd5sum, probe_with, quiet
+#- options: callback, ask_retry, force, nomd5sum, probe_with, quiet
 sub _update_medium__parse_if_unmodified__remote {
     my ($urpm, $medium, $options) = @_;
 
@@ -1563,7 +1563,7 @@ sub _read_cachedir_pubkey {
     join(',', keys %key_ids);
 }
 
-#- options: callback, force, nomd5sum, probe_with, quiet, forcekey, nopubkey, wait_lock
+#- options: callback, ask_retry, force, nomd5sum, probe_with, quiet, forcekey, nopubkey, wait_lock
 #- (from _update_medium__parse_if_unmodified__local and _update_medium__parse_if_unmodified__remote)
 sub _update_medium_ {
     my ($urpm, $medium, %options) = @_;
@@ -1664,6 +1664,7 @@ sub _update_media__handle_some_flags {
 #- the recomputation of base files.
 #- Recognized options :
 #-   all         : all medias are being rebuilt
+#-   ask_retry   : function called when a download fails. if it returns true, the download is retried
 #-   callback    : UI callback
 #-   forcekey    : force retrieval of pubkey
 #-   force       : try to force rebuilding base files