From 14abfdf01bfa2e0872a85ea7ce022d44a6e29fb7 Mon Sep 17 00:00:00 2001
From: Pascal Rigaux <pixel@mandriva.com>
Date: Sat, 23 Feb 2008 21:31:09 +0000
Subject: - all tools:   o handle mirrorlist     (need perl-Zone-TimeInfo
 patched for geolocalisation)

---
 urpm/args.pm    |   1 +
 urpm/cfg.pm     |   6 +-
 urpm/media.pm   |  85 +++++++++++++++++++----
 urpm/mirrors.pm | 212 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 288 insertions(+), 16 deletions(-)
 create mode 100644 urpm/mirrors.pm

(limited to 'urpm')

diff --git a/urpm/args.pm b/urpm/args.pm
index d809ee9d..73665d9d 100644
--- a/urpm/args.pm
+++ b/urpm/args.pm
@@ -316,6 +316,7 @@ my %options_spec = (
 	'xml-info=s' => \$options{'xml-info'},
 	'no-probe' => sub { $options{probe_with} = undef },
 	distrib => sub { $options{distrib} = 1 },
+	'mirrorlist=s' => \$options{mirrorlist},
         interactive => sub { $options{interactive} = 1 },
         'all-media' => sub { $options{allmedia} = 1 },
 	'from=s' => \$options{mirrors_url},
diff --git a/urpm/cfg.pm b/urpm/cfg.pm
index 03175d5f..3283c6e7 100644
--- a/urpm/cfg.pm
+++ b/urpm/cfg.pm
@@ -140,11 +140,13 @@ sub load_config_raw {
 		$err = N("medium `%s' is defined twice, aborting", $name);
 		return;
 	    }
-	    $block = { name => $name, url => $url };
+	    $block = { name => $name, $url ? (url => $url) : () };
 	} elsif (/^(hdlist
 	  |list
 	  |with_hdlist
 	  |with_synthesis
+	  |with-dir
+          |mirrorlist
 	  |media_info_dir
 	  |removable
 	  |md5sum
@@ -294,7 +296,7 @@ sub write_ini_config {
     foreach (@$blocks) {
 	my %h = %$_;
 	my $section = delete $h{'with-dir'} || '_';
-	$uniq{$section}++ or die "conflicting with-dir value\n";
+	$uniq{$section}++ or die "conflicting with-dir value $section\n";
 
 	foreach (difference2([ $cfg->Parameters($section) ], [ keys %h ])) {
 	    # remove those options which are no more wanted
diff --git a/urpm/media.pm b/urpm/media.pm
index 2e91fdd5..c7995b52 100644
--- a/urpm/media.pm
+++ b/urpm/media.pm
@@ -16,6 +16,7 @@ our @PER_MEDIA_OPT = qw(
     key-ids
     list
     media_info_dir
+    mirrorlist
     name
     no-media-info
     noreconfigure
@@ -56,6 +57,7 @@ sub _only_media_opts_write {
     my ($m) = @_;
     my $c = only_media_opts($m);
     delete $c->{media_info_dir} if $c->{media_info_dir} eq 'media_info';
+    delete $c->{url} if $c->{mirrorlist};
     $c;
 }
 
@@ -84,7 +86,7 @@ sub read_config_add_passwords {
     my ($urpm, $config) = @_;
 
     my @netrc = read_private_netrc($urpm) or return;
-    foreach (@{$config->{media}}) {
+    foreach (grep { $_->{url} } @{$config->{media}}) {
 	my $u = urpm::download::parse_url_with_login($_->{url}) or next;
 	if (my ($e) = grep { ($_->{default} || $_->{machine} eq $u->{machine}) && $_->{login} eq $u->{login} } @netrc) {
 	    $_->{url} = sprintf('%s://%s:%s@%s%s', $u->{proto}, $u->{login}, $e->{password}, $u->{machine}, $u->{dir});
@@ -98,7 +100,7 @@ sub remove_passwords_and_write_private_netrc {
     my ($urpm, $config) = @_;
 
     my @l;
-    foreach (@{$config->{media}}) {
+    foreach (grep { $_->{url} } @{$config->{media}}) {
 	my $u = urpm::download::parse_url_with_login($_->{url}) or next;
 	#- check whether a password is visible
 	$u->{password} or next;
@@ -149,7 +151,7 @@ sub read_config {
     foreach my $m (@{$config->{media}}) {
 	my $medium = _only_media_opts_read($m);
 
-	if (!$medium->{url}) {
+	if (!$medium->{url} && !$medium->{mirrorlist}) {
 	    #- recover the url the old deprecated way...
 	    #- only useful for migration, new urpmi.cfg will use netrc
 	    recover_url_from_list($urpm, $medium);
@@ -180,7 +182,7 @@ sub check_existing_medium {
     my ($urpm, $medium) = @_;
 
     my $err;
-    if (!$medium->{url}) { 
+    if (!$medium->{url} && !$medium->{mirrorlist}) { 
 	$err = $medium->{virtual} ?
 	  N("virtual medium \"%s\" should have a clear url, medium ignored",
 			   $medium->{name}) :
@@ -602,6 +604,8 @@ sub _parse_media {
 	delete @$_{qw(start end)};
 	_parse_synthesis_or_ignore($urpm, $_, $options->{callback});
 
+	_pick_mirror_if_needed($urpm, $_, '');
+
 	if ($_->{searchmedia}) {
 	    $urpm->{searchmedia} = 1;
 	    $urpm->{log}(N("Search start: %s end: %s", $_->{start}, $_->{end}));
@@ -644,7 +648,7 @@ sub _compute_flags_for_instlist {
 #- add a new medium, sync the config file accordingly.
 #- returns the new medium's name. (might be different from the requested
 #- name if index_name was specified)
-#- options: ignore, index_name, nolock, update, virtual, media_info_dir, xml-info
+#- options: ignore, index_name, nolock, update, virtual, media_info_dir, mirrorlist, with-dir, xml-info
 sub add_medium {
     my ($urpm, $name, $url, $with_synthesis, %options) = @_;
 
@@ -670,7 +674,7 @@ sub add_medium {
 		url => $url, 
 		modified => !$options{ignore}, 
 	    };
-    foreach (qw(downloader update ignore media_info_dir xml-info)) {
+    foreach (qw(downloader update ignore media_info_dir mirrorlist with-dir xml-info)) {
 	$medium->{$_} = $options{$_} if exists $options{$_};
     }
 
@@ -683,6 +687,17 @@ sub add_medium {
 	probe_removable_device($urpm, $medium);
     }
 
+    if (!$medium->{url} && $options{mirrorlist}) {	
+	# forcing the standard media_info_dir if undefined
+	$medium->{media_info_dir} ||= 'media_info';
+
+	require urpm::mirrors;
+	urpm::mirrors::try($urpm, $medium, sub {
+	    # this is a little ugly since MD5SUM will be downloaded again later, but it's small enough...
+	    _download_MD5SUM($urpm, $medium);
+	}) or return;
+    }
+
     if ($with_synthesis) {
 	_migrate__with_synthesis($medium, $with_synthesis);
     } elsif (!$medium->{media_info_dir}) {
@@ -719,6 +734,7 @@ sub add_medium {
 #- - probe_with : force use of rpms instead of using synthesis
 #- - ask_media : callback to know whether each media should be added
 #- - only_updates : only add "update" media (used by rpmdrake)
+#- - mirrorlist
 #- other options are passed to add_medium(): ignore, nolock, virtual
 sub add_distrib_media {
     my ($urpm, $name, $url, %options) = @_;
@@ -728,7 +744,7 @@ sub add_distrib_media {
 
     my $distribconf;
 
-    if (my $dir = file_from_local_url($url)) {
+    if (my $dir = $url && file_from_local_url($url)) {
 	urpm::removable::try_mounting($urpm, $dir)
 	    or $urpm->{error}(N("unable to mount the distribution medium")), return ();
 	$distribconf = MDV::Distribconf->new($dir, undef);
@@ -737,7 +753,15 @@ sub add_distrib_media {
     } else {
 	unlink "$urpm->{cachedir}/partial/media.cfg";
 
-	$distribconf = _new_distribconf_and_download($urpm, $url);
+	if ($options{mirrorlist}) {
+	    $url and die "unexpected url $url together with mirrorlist $options{mirrorlist}\n";
+	}
+
+	my $m = { mirrorlist => $options{mirrorlist}, url => $url };
+	try__maybe_mirrorlist($urpm, $m, sub {
+	    $distribconf = _new_distribconf_and_download($urpm, $m->{url});
+	});
+	$url = $m->{url};
 
 	if ($distribconf) {
 	    $distribconf->parse_mediacfg("$urpm->{cachedir}/partial/media.cfg")
@@ -795,6 +819,7 @@ sub add_distrib_media {
 	    !$use_copied_synthesis && $options{probe_with} ? ($options{probe_with} => 1) : (),
 	    index_name => $name ? undef : 0,
 	    $add_by_default ? () : (ignore => 1),
+	    $options{mirrorlist} ? ('with-dir' => $distribconf->getpath($media, 'path')) : (),
 	    %options,
 	    # the following override %options
 	    update => $is_update_media ? 1 : undef,
@@ -900,6 +925,8 @@ sub _clean_statedir_medium_files {
 sub _probe_with_try_list {
     my ($urpm, $medium, $f) = @_;
 
+    $medium->{mirrorlist} and die "_probe_with_try_list does not handle mirrorlist\n";
+
     my @media_info_dirs = ('media_info', '.');
 
     my $base = file_from_local_url($medium->{url}) || $medium->{url};
@@ -919,6 +946,8 @@ sub _probe_with_try_list {
 sub may_reconfig_urpmi {
     my ($urpm, $medium) = @_;
 
+    $medium->{url} or return; # we should handle mirrorlist?
+
     my $f;
     if (my $dir = file_from_local_url($medium->{url})) {
 	$f = reduce_pathname("$dir/reconfig.urpmi");
@@ -1116,6 +1145,7 @@ sub get_descriptions_local {
 	$medium->{ignore} = 1;
     }
 }
+#- not handling different mirrors since the file is not always available
 sub get_descriptions_remote {
     my ($urpm, $medium) = @_;
 
@@ -1158,15 +1188,16 @@ sub get_synthesis__local {
 sub get_synthesis__remote {
     my ($urpm, $medium, $callback, $quiet) = @_;
 
-    if (urpm::download::sync($urpm, $medium, [ _url_with_synthesis($medium) ],
+    my $ok = try__maybe_mirrorlist($urpm, $medium, sub {
+	urpm::download::sync($urpm, $medium, [ _url_with_synthesis($medium) ],
 			     quiet => $quiet, callback => $callback) &&
-			       file_size(cachedir_with_synthesis($urpm, $medium)) >= 20) {
-	1;
-    } else {
+			       file_size(cachedir_with_synthesis($urpm, $medium)) >= 20;
+    });
+    if (!$ok) {
 	chomp(my $err = $@);
 	$urpm->{error}(N("...retrieving failed: %s", $err));
-	0;
     }
+    $ok;
 }
 
 #- check copied/downloaded file has right signature.
@@ -1425,6 +1456,8 @@ sub _update_medium_ {
 	unlink "$urpm->{cachedir}/partial/$_";
     }
 
+    _pick_mirror_if_needed($urpm, $medium, 'allow-cache-update');
+
     #- check for a reconfig.urpmi file (if not already reconfigured)
     if (!$medium->{noreconfigure}) {
 	may_reconfig_urpmi($urpm, $medium);
@@ -1503,7 +1536,7 @@ sub _update_media__handle_some_flags {
 	    $medium->{modified} = 0;
 	} elsif ($all) {
 	    #- if we're rebuilding all media, mark them as modified (except removable ones)
-	    $medium->{modified} ||= $medium->{url} !~ m!^removable!;
+	    $medium->{modified} ||= !($medium->{url} && $medium->{url} =~ m!^removable!);
 	}
     }
 }
@@ -1625,6 +1658,30 @@ sub _any_media_info__or_download {
     $f;
 }
 
+#- side-effects:
+#-   + those of urpm::mirrors::pick_one ($urpm->{mirrors_cache}, $medium->{url})
+sub _pick_mirror_if_needed {
+    my ($urpm, $medium, $allow_cache_update) = @_;
+
+    $medium->{mirrorlist} && !$medium->{url} or return;
+
+    require urpm::mirrors;
+    urpm::mirrors::pick_one($urpm, $medium, $allow_cache_update);
+}
+
+#- side-effects:
+#-   + those of urpm::mirrors::try ($urpm->{mirrors_cache}, $medium->{url})
+sub try__maybe_mirrorlist {
+    my ($urpm, $medium, $try) = @_;
+
+    if ($medium->{mirrorlist}) {
+	require urpm::mirrors;
+	urpm::mirrors::try($urpm, $medium, $try);
+    } else {
+	$try->();
+    }
+}
+
 #- clean params and depslist computation zone.
 sub clean {
     my ($urpm) = @_;
diff --git a/urpm/mirrors.pm b/urpm/mirrors.pm
new file mode 100644
index 00000000..17366b38
--- /dev/null
+++ b/urpm/mirrors.pm
@@ -0,0 +1,212 @@
+package urpm::mirrors;
+
+# $Id: $
+
+use strict;
+use urpm::util;
+use urpm::msg;
+use urpm::download;
+
+
+#- $medium fields used: mirrorlist, with-dir
+#- side-effects: $medium->{url}
+#-   + those of _pick_one ($urpm->{mirrors_cache})
+sub try {
+    my ($urpm, $medium, $try) = @_;
+
+    for (my $nb = 1; $nb < $urpm->{options}{'max-round-robin-tries'}; $nb++) {
+	my $url = _pick_one($urpm, $medium->{mirrorlist}, $nb == 1, '') or return;
+	$urpm->{info}(N("trying again with mirror %s", $url)) if $nb > 1;
+	$medium->{url} = _add__with_dir($url, $medium->{'with-dir'});
+	$try->() and return 1;
+	black_list($urpm, $medium->{mirrorlist}, $url);
+    }
+    0;
+}
+
+#- side-effects: none
+sub _add__with_dir {
+    my ($url, $with_dir) = @_;
+    reduce_pathname($url . ($with_dir ? "/$with_dir" : ''));
+}
+
+#- side-effects: $medium->{url}
+#-   + those of _pick_one ($urpm->{mirrors_cache})
+sub pick_one {
+    my ($urpm, $medium, $allow_cache_update) = @_;   
+
+    my $url = _pick_one($urpm, $medium->{mirrorlist}, 'must_succeed', $allow_cache_update);
+    $medium->{url} = _add__with_dir($url, $medium->{'with-dir'});
+}
+
+#- side-effects: $urpm->{mirrors_cache}
+sub _pick_one {
+    my ($urpm, $mirrorlist, $must_succeed, $allow_cache_update) = @_;   
+    my $cache = _cache($urpm, $mirrorlist);
+
+    if ($allow_cache_update && $cache->{time} &&
+	  time() > $cache->{time} + 24*60*60 * $urpm->{options}{'days-between-mirrorlist-update'}) {
+	$urpm->{log}("not using outdated cached mirror list");
+	%$cache = ();
+    }
+
+    if (!$cache->{chosen}) {
+	if (!$cache->{list}) {
+	    $cache->{list} = [ _list($urpm, $mirrorlist) ];
+	    $cache->{time} = time();
+	}
+
+	$cache->{chosen} = $cache->{list}[0]{url} or do {
+	    $must_succeed and $urpm->{fatal}(10, N("Could not find a mirror from mirrorlist %s", $mirrorlist));
+	    return;
+	};
+	_save_cache($urpm);
+    }
+    if ($cache->{nb_uses}++) {
+	$urpm->{debug} and $urpm->{debug}("using mirror $cache->{chosen}");
+    } else {
+	$urpm->{log}("using mirror $cache->{chosen}");
+    }
+
+    $cache->{chosen};
+}
+#- side-effects: $urpm->{mirrors_cache}
+sub black_list {
+    my ($urpm, $mirrorlist, $url) = @_;
+    my $cache = _cache($urpm, $mirrorlist);
+
+    @{$cache->{list}} = grep { $_->{url} ne $url } @{$cache->{list}};
+    delete $cache->{chosen};
+}
+#- side-effects: $urpm->{mirrors_cache}
+sub _cache {
+    my ($urpm, $mirrorlist) = @_;
+    my $full_cache = $urpm->{mirrors_cache} ||= _load_cache($urpm);
+    $full_cache->{$mirrorlist} ||= {};
+}
+sub cache_file {
+    my ($urpm) = @_;
+    my $cache_file = "$urpm->{cachedir}/mirrors.cache";
+}
+sub _load_cache {
+    my ($urpm) = @_;
+    my $cache;
+    if (-e cache_file($urpm)) {
+	$urpm->{debug} and $urpm->{debug}("loading mirrors cache");
+	$cache = eval(cat_(cache_file($urpm)));
+	$@ and $urpm->{error}("failed to read " . cache_file($urpm) . ": $@");
+	$_->{nb_uses} = 0 foreach values %$cache;
+    }
+    $cache || {};
+}
+sub _save_cache {
+    my ($urpm) = @_;
+    require Data::Dumper;
+    my $s = Data::Dumper::Dumper($urpm->{mirrors_cache});
+    $s =~ s/.*?=//; # get rid of $VAR1 = 
+    output_safe(cache_file($urpm), $s);
+}
+
+#- side-effects: none
+sub _list {
+    my ($urpm, $mirrorlist) = @_;
+
+    # expand the variable
+    $mirrorlist = _MIRRORLIST() if $mirrorlist eq '$MIRRORLIST';
+
+    my @mirrors = _mirrors_filtered($urpm, $mirrorlist);
+    add_proximity_and_sort($urpm, \@mirrors);
+    @mirrors;
+}
+
+#- side-effects: $mirrors
+sub add_proximity_and_sort {
+    my ($urpm, $mirrors) = @_;
+
+    my ($latitude, $longitude, $country_code);
+
+    require Time::ZoneInfo;
+    if (my $zone = Time::ZoneInfo->current_zone) {
+	if (my $zones = Time::ZoneInfo->new) {
+	    if (($latitude, $longitude) = $zones->latitude_longitude_decimal($zone)) {
+		$country_code = $zones->country($zone);
+		$urpm->{log}(N("found geolocalisation %s %.2f %.2f from timezone %s", $country_code, $latitude, $longitude, $zone));
+	    }
+	}
+    }
+    defined $latitude && defined $longitude or return;
+
+    foreach (@$mirrors) {
+	$_->{latitude} || $_->{longitude} or next;
+	my $PI = 3.14159265358979;
+	my $x = $latitude - $_->{latitude};
+	my $y = ($longitude - $_->{longitude}) * cos($_->{latitude} / 180 * $PI);
+	$_->{proximity} = sqrt($x * $x + $y * $y);
+    }
+    my ($best) = sort { $a->{proximity} <=> $b->{proximity} } @$mirrors;
+
+    foreach (@$mirrors) {
+	$_->{proximity_corrected} = $_->{proximity} * _random_correction();
+	$_->{proximity_corrected} *= _between_country_correction($country_code, $_->{country}) if $best;
+	$_->{proximity_corrected} *= _between_continent_correction($best->{continent}, $_->{continent}) if $best;
+    }
+    @$mirrors = sort { $a->{proximity_corrected} <=> $b->{proximity_corrected} } @$mirrors;
+}
+
+# add +/- 5% random
+sub _random_correction() {
+    my $correction = 0.05;
+    1 + (rand() - 0.5) * $correction * 2;
+}
+
+sub _between_country_correction {
+    my ($here, $mirror) = @_;
+    $here && $mirror or return 1;
+    $here eq $mirror ? 0.5 : 1;
+}
+sub _between_continent_correction {
+    my ($here, $mirror) = @_;
+    $here && $mirror or return 1;
+    $here eq $mirror ? 0.5 : # favor same continent
+      $here eq 'SA' && $mirror eq 'NA' ? 0.9 : # favor going "South America" -> "North America"
+	1;
+}
+
+sub _mirrors_raw {
+    my ($urpm, $url) = @_;
+
+    $urpm->{log}(N("getting mirror list from %s", $url));
+    my @l = urpm::download::get_content($urpm, $url) or die "mirror list not found";
+    @l;
+}
+
+sub _mirrors_filtered {
+    my ($urpm, $mirrorlist) = @_;
+
+    grep {
+	$_->{type} eq 'distrib'; # type=updates seems to be history, and type=iso is not interesting here
+    } map { chomp; parse_LDAP_namespace_structure($_) } _mirrors_raw($urpm, $mirrorlist);
+}
+
+sub _MIRRORLIST() {
+    my $product_id = parse_LDAP_namespace_structure(cat_('/etc/product.id'));
+    _mandriva_mirrorlist($product_id);
+}
+sub _mandriva_mirrorlist {
+    my ($product_id, $o_arch) = @_;
+
+    #- contact the following URL to retrieve the list of mirrors.
+    #- http://wiki.mandriva.com/en/Product_id
+    my $product_type = lc($product_id->{type}); $product_id =~ s/\s//g;
+    my $arch = $o_arch || $product_id->{arch};
+
+    "http://api.mandriva.com/mirrors/$product_type.$product_id->{version}.$arch.list";
+}
+
+sub parse_LDAP_namespace_structure {
+    my ($s) = @_;
+    my %h = map { /(.*?)=(.*)/ ? ($1 => $2) : () } split(',', $s);
+    \%h;
+}
+
+1;
-- 
cgit v1.2.1