summaryrefslogtreecommitdiffstats
path: root/urpm
diff options
context:
space:
mode:
Diffstat (limited to 'urpm')
-rw-r--r--urpm/args.pm1
-rw-r--r--urpm/cfg.pm6
-rw-r--r--urpm/media.pm85
-rw-r--r--urpm/mirrors.pm212
4 files changed, 288 insertions, 16 deletions
diff --git a/urpm/args.pm b/urpm/args.pm
index d809ee9d..73665d9d 100644
--- a/urpm/args.pm
+++ b/urpm/args.pm
@@ -316,6 +316,7 @@ my %options_spec = (
'xml-info=s' => \$options{'xml-info'},
'no-probe' => sub { $options{probe_with} = undef },
distrib => sub { $options{distrib} = 1 },
+ 'mirrorlist=s' => \$options{mirrorlist},
interactive => sub { $options{interactive} = 1 },
'all-media' => sub { $options{allmedia} = 1 },
'from=s' => \$options{mirrors_url},
diff --git a/urpm/cfg.pm b/urpm/cfg.pm
index 03175d5f..3283c6e7 100644
--- a/urpm/cfg.pm
+++ b/urpm/cfg.pm
@@ -140,11 +140,13 @@ sub load_config_raw {
$err = N("medium `%s' is defined twice, aborting", $name);
return;
}
- $block = { name => $name, url => $url };
+ $block = { name => $name, $url ? (url => $url) : () };
} elsif (/^(hdlist
|list
|with_hdlist
|with_synthesis
+ |with-dir
+ |mirrorlist
|media_info_dir
|removable
|md5sum
@@ -294,7 +296,7 @@ sub write_ini_config {
foreach (@$blocks) {
my %h = %$_;
my $section = delete $h{'with-dir'} || '_';
- $uniq{$section}++ or die "conflicting with-dir value\n";
+ $uniq{$section}++ or die "conflicting with-dir value $section\n";
foreach (difference2([ $cfg->Parameters($section) ], [ keys %h ])) {
# remove those options which are no more wanted
diff --git a/urpm/media.pm b/urpm/media.pm
index 2e91fdd5..c7995b52 100644
--- a/urpm/media.pm
+++ b/urpm/media.pm
@@ -16,6 +16,7 @@ our @PER_MEDIA_OPT = qw(
key-ids
list
media_info_dir
+ mirrorlist
name
no-media-info
noreconfigure
@@ -56,6 +57,7 @@ sub _only_media_opts_write {
my ($m) = @_;
my $c = only_media_opts($m);
delete $c->{media_info_dir} if $c->{media_info_dir} eq 'media_info';
+ delete $c->{url} if $c->{mirrorlist};
$c;
}
@@ -84,7 +86,7 @@ sub read_config_add_passwords {
my ($urpm, $config) = @_;
my @netrc = read_private_netrc($urpm) or return;
- foreach (@{$config->{media}}) {
+ foreach (grep { $_->{url} } @{$config->{media}}) {
my $u = urpm::download::parse_url_with_login($_->{url}) or next;
if (my ($e) = grep { ($_->{default} || $_->{machine} eq $u->{machine}) && $_->{login} eq $u->{login} } @netrc) {
$_->{url} = sprintf('%s://%s:%s@%s%s', $u->{proto}, $u->{login}, $e->{password}, $u->{machine}, $u->{dir});
@@ -98,7 +100,7 @@ sub remove_passwords_and_write_private_netrc {
my ($urpm, $config) = @_;
my @l;
- foreach (@{$config->{media}}) {
+ foreach (grep { $_->{url} } @{$config->{media}}) {
my $u = urpm::download::parse_url_with_login($_->{url}) or next;
#- check whether a password is visible
$u->{password} or next;
@@ -149,7 +151,7 @@ sub read_config {
foreach my $m (@{$config->{media}}) {
my $medium = _only_media_opts_read($m);
- if (!$medium->{url}) {
+ if (!$medium->{url} && !$medium->{mirrorlist}) {
#- recover the url the old deprecated way...
#- only useful for migration, new urpmi.cfg will use netrc
recover_url_from_list($urpm, $medium);
@@ -180,7 +182,7 @@ sub check_existing_medium {
my ($urpm, $medium) = @_;
my $err;
- if (!$medium->{url}) {
+ if (!$medium->{url} && !$medium->{mirrorlist}) {
$err = $medium->{virtual} ?
N("virtual medium \"%s\" should have a clear url, medium ignored",
$medium->{name}) :
@@ -602,6 +604,8 @@ sub _parse_media {
delete @$_{qw(start end)};
_parse_synthesis_or_ignore($urpm, $_, $options->{callback});
+ _pick_mirror_if_needed($urpm, $_, '');
+
if ($_->{searchmedia}) {
$urpm->{searchmedia} = 1;
$urpm->{log}(N("Search start: %s end: %s", $_->{start}, $_->{end}));
@@ -644,7 +648,7 @@ sub _compute_flags_for_instlist {
#- add a new medium, sync the config file accordingly.
#- returns the new medium's name. (might be different from the requested
#- name if index_name was specified)
-#- options: ignore, index_name, nolock, update, virtual, media_info_dir, xml-info
+#- options: ignore, index_name, nolock, update, virtual, media_info_dir, mirrorlist, with-dir, xml-info
sub add_medium {
my ($urpm, $name, $url, $with_synthesis, %options) = @_;
@@ -670,7 +674,7 @@ sub add_medium {
url => $url,
modified => !$options{ignore},
};
- foreach (qw(downloader update ignore media_info_dir xml-info)) {
+ foreach (qw(downloader update ignore media_info_dir mirrorlist with-dir xml-info)) {
$medium->{$_} = $options{$_} if exists $options{$_};
}
@@ -683,6 +687,17 @@ sub add_medium {
probe_removable_device($urpm, $medium);
}
+ if (!$medium->{url} && $options{mirrorlist}) {
+ # forcing the standard media_info_dir if undefined
+ $medium->{media_info_dir} ||= 'media_info';
+
+ require urpm::mirrors;
+ urpm::mirrors::try($urpm, $medium, sub {
+ # this is a little ugly since MD5SUM will be downloaded again later, but it's small enough...
+ _download_MD5SUM($urpm, $medium);
+ }) or return;
+ }
+
if ($with_synthesis) {
_migrate__with_synthesis($medium, $with_synthesis);
} elsif (!$medium->{media_info_dir}) {
@@ -719,6 +734,7 @@ sub add_medium {
#- - probe_with : force use of rpms instead of using synthesis
#- - ask_media : callback to know whether each media should be added
#- - only_updates : only add "update" media (used by rpmdrake)
+#- - mirrorlist
#- other options are passed to add_medium(): ignore, nolock, virtual
sub add_distrib_media {
my ($urpm, $name, $url, %options) = @_;
@@ -728,7 +744,7 @@ sub add_distrib_media {
my $distribconf;
- if (my $dir = file_from_local_url($url)) {
+ if (my $dir = $url && file_from_local_url($url)) {
urpm::removable::try_mounting($urpm, $dir)
or $urpm->{error}(N("unable to mount the distribution medium")), return ();
$distribconf = MDV::Distribconf->new($dir, undef);
@@ -737,7 +753,15 @@ sub add_distrib_media {
} else {
unlink "$urpm->{cachedir}/partial/media.cfg";
- $distribconf = _new_distribconf_and_download($urpm, $url);
+ if ($options{mirrorlist}) {
+ $url and die "unexpected url $url together with mirrorlist $options{mirrorlist}\n";
+ }
+
+ my $m = { mirrorlist => $options{mirrorlist}, url => $url };
+ try__maybe_mirrorlist($urpm, $m, sub {
+ $distribconf = _new_distribconf_and_download($urpm, $m->{url});
+ });
+ $url = $m->{url};
if ($distribconf) {
$distribconf->parse_mediacfg("$urpm->{cachedir}/partial/media.cfg")
@@ -795,6 +819,7 @@ sub add_distrib_media {
!$use_copied_synthesis && $options{probe_with} ? ($options{probe_with} => 1) : (),
index_name => $name ? undef : 0,
$add_by_default ? () : (ignore => 1),
+ $options{mirrorlist} ? ('with-dir' => $distribconf->getpath($media, 'path')) : (),
%options,
# the following override %options
update => $is_update_media ? 1 : undef,
@@ -900,6 +925,8 @@ sub _clean_statedir_medium_files {
sub _probe_with_try_list {
my ($urpm, $medium, $f) = @_;
+ $medium->{mirrorlist} and die "_probe_with_try_list does not handle mirrorlist\n";
+
my @media_info_dirs = ('media_info', '.');
my $base = file_from_local_url($medium->{url}) || $medium->{url};
@@ -919,6 +946,8 @@ sub _probe_with_try_list {
sub may_reconfig_urpmi {
my ($urpm, $medium) = @_;
+ $medium->{url} or return; # we should handle mirrorlist?
+
my $f;
if (my $dir = file_from_local_url($medium->{url})) {
$f = reduce_pathname("$dir/reconfig.urpmi");
@@ -1116,6 +1145,7 @@ sub get_descriptions_local {
$medium->{ignore} = 1;
}
}
+#- not handling different mirrors since the file is not always available
sub get_descriptions_remote {
my ($urpm, $medium) = @_;
@@ -1158,15 +1188,16 @@ sub get_synthesis__local {
sub get_synthesis__remote {
my ($urpm, $medium, $callback, $quiet) = @_;
- if (urpm::download::sync($urpm, $medium, [ _url_with_synthesis($medium) ],
+ my $ok = try__maybe_mirrorlist($urpm, $medium, sub {
+ urpm::download::sync($urpm, $medium, [ _url_with_synthesis($medium) ],
quiet => $quiet, callback => $callback) &&
- file_size(cachedir_with_synthesis($urpm, $medium)) >= 20) {
- 1;
- } else {
+ file_size(cachedir_with_synthesis($urpm, $medium)) >= 20;
+ });
+ if (!$ok) {
chomp(my $err = $@);
$urpm->{error}(N("...retrieving failed: %s", $err));
- 0;
}
+ $ok;
}
#- check copied/downloaded file has right signature.
@@ -1425,6 +1456,8 @@ sub _update_medium_ {
unlink "$urpm->{cachedir}/partial/$_";
}
+ _pick_mirror_if_needed($urpm, $medium, 'allow-cache-update');
+
#- check for a reconfig.urpmi file (if not already reconfigured)
if (!$medium->{noreconfigure}) {
may_reconfig_urpmi($urpm, $medium);
@@ -1503,7 +1536,7 @@ sub _update_media__handle_some_flags {
$medium->{modified} = 0;
} elsif ($all) {
#- if we're rebuilding all media, mark them as modified (except removable ones)
- $medium->{modified} ||= $medium->{url} !~ m!^removable!;
+ $medium->{modified} ||= !($medium->{url} && $medium->{url} =~ m!^removable!);
}
}
}
@@ -1625,6 +1658,30 @@ sub _any_media_info__or_download {
$f;
}
+#- side-effects:
+#- + those of urpm::mirrors::pick_one ($urpm->{mirrors_cache}, $medium->{url})
+sub _pick_mirror_if_needed {
+ my ($urpm, $medium, $allow_cache_update) = @_;
+
+ $medium->{mirrorlist} && !$medium->{url} or return;
+
+ require urpm::mirrors;
+ urpm::mirrors::pick_one($urpm, $medium, $allow_cache_update);
+}
+
+#- side-effects:
+#- + those of urpm::mirrors::try ($urpm->{mirrors_cache}, $medium->{url})
+sub try__maybe_mirrorlist {
+ my ($urpm, $medium, $try) = @_;
+
+ if ($medium->{mirrorlist}) {
+ require urpm::mirrors;
+ urpm::mirrors::try($urpm, $medium, $try);
+ } else {
+ $try->();
+ }
+}
+
#- clean params and depslist computation zone.
sub clean {
my ($urpm) = @_;
diff --git a/urpm/mirrors.pm b/urpm/mirrors.pm
new file mode 100644
index 00000000..17366b38
--- /dev/null
+++ b/urpm/mirrors.pm
@@ -0,0 +1,212 @@
+package urpm::mirrors;
+
+# $Id: $
+
+use strict;
+use urpm::util;
+use urpm::msg;
+use urpm::download;
+
+
+#- $medium fields used: mirrorlist, with-dir
+#- side-effects: $medium->{url}
+#- + those of _pick_one ($urpm->{mirrors_cache})
+sub try {
+ my ($urpm, $medium, $try) = @_;
+
+ for (my $nb = 1; $nb < $urpm->{options}{'max-round-robin-tries'}; $nb++) {
+ my $url = _pick_one($urpm, $medium->{mirrorlist}, $nb == 1, '') or return;
+ $urpm->{info}(N("trying again with mirror %s", $url)) if $nb > 1;
+ $medium->{url} = _add__with_dir($url, $medium->{'with-dir'});
+ $try->() and return 1;
+ black_list($urpm, $medium->{mirrorlist}, $url);
+ }
+ 0;
+}
+
+#- side-effects: none
+sub _add__with_dir {
+ my ($url, $with_dir) = @_;
+ reduce_pathname($url . ($with_dir ? "/$with_dir" : ''));
+}
+
+#- side-effects: $medium->{url}
+#- + those of _pick_one ($urpm->{mirrors_cache})
+sub pick_one {
+ my ($urpm, $medium, $allow_cache_update) = @_;
+
+ my $url = _pick_one($urpm, $medium->{mirrorlist}, 'must_succeed', $allow_cache_update);
+ $medium->{url} = _add__with_dir($url, $medium->{'with-dir'});
+}
+
+#- side-effects: $urpm->{mirrors_cache}
+sub _pick_one {
+ my ($urpm, $mirrorlist, $must_succeed, $allow_cache_update) = @_;
+ my $cache = _cache($urpm, $mirrorlist);
+
+ if ($allow_cache_update && $cache->{time} &&
+ time() > $cache->{time} + 24*60*60 * $urpm->{options}{'days-between-mirrorlist-update'}) {
+ $urpm->{log}("not using outdated cached mirror list");
+ %$cache = ();
+ }
+
+ if (!$cache->{chosen}) {
+ if (!$cache->{list}) {
+ $cache->{list} = [ _list($urpm, $mirrorlist) ];
+ $cache->{time} = time();
+ }
+
+ $cache->{chosen} = $cache->{list}[0]{url} or do {
+ $must_succeed and $urpm->{fatal}(10, N("Could not find a mirror from mirrorlist %s", $mirrorlist));
+ return;
+ };
+ _save_cache($urpm);
+ }
+ if ($cache->{nb_uses}++) {
+ $urpm->{debug} and $urpm->{debug}("using mirror $cache->{chosen}");
+ } else {
+ $urpm->{log}("using mirror $cache->{chosen}");
+ }
+
+ $cache->{chosen};
+}
+#- side-effects: $urpm->{mirrors_cache}
+sub black_list {
+ my ($urpm, $mirrorlist, $url) = @_;
+ my $cache = _cache($urpm, $mirrorlist);
+
+ @{$cache->{list}} = grep { $_->{url} ne $url } @{$cache->{list}};
+ delete $cache->{chosen};
+}
+#- side-effects: $urpm->{mirrors_cache}
+sub _cache {
+ my ($urpm, $mirrorlist) = @_;
+ my $full_cache = $urpm->{mirrors_cache} ||= _load_cache($urpm);
+ $full_cache->{$mirrorlist} ||= {};
+}
+sub cache_file {
+ my ($urpm) = @_;
+ my $cache_file = "$urpm->{cachedir}/mirrors.cache";
+}
+sub _load_cache {
+ my ($urpm) = @_;
+ my $cache;
+ if (-e cache_file($urpm)) {
+ $urpm->{debug} and $urpm->{debug}("loading mirrors cache");
+ $cache = eval(cat_(cache_file($urpm)));
+ $@ and $urpm->{error}("failed to read " . cache_file($urpm) . ": $@");
+ $_->{nb_uses} = 0 foreach values %$cache;
+ }
+ $cache || {};
+}
+sub _save_cache {
+ my ($urpm) = @_;
+ require Data::Dumper;
+ my $s = Data::Dumper::Dumper($urpm->{mirrors_cache});
+ $s =~ s/.*?=//; # get rid of $VAR1 =
+ output_safe(cache_file($urpm), $s);
+}
+
+#- side-effects: none
+sub _list {
+ my ($urpm, $mirrorlist) = @_;
+
+ # expand the variable
+ $mirrorlist = _MIRRORLIST() if $mirrorlist eq '$MIRRORLIST';
+
+ my @mirrors = _mirrors_filtered($urpm, $mirrorlist);
+ add_proximity_and_sort($urpm, \@mirrors);
+ @mirrors;
+}
+
+#- side-effects: $mirrors
+sub add_proximity_and_sort {
+ my ($urpm, $mirrors) = @_;
+
+ my ($latitude, $longitude, $country_code);
+
+ require Time::ZoneInfo;
+ if (my $zone = Time::ZoneInfo->current_zone) {
+ if (my $zones = Time::ZoneInfo->new) {
+ if (($latitude, $longitude) = $zones->latitude_longitude_decimal($zone)) {
+ $country_code = $zones->country($zone);
+ $urpm->{log}(N("found geolocalisation %s %.2f %.2f from timezone %s", $country_code, $latitude, $longitude, $zone));
+ }
+ }
+ }
+ defined $latitude && defined $longitude or return;
+
+ foreach (@$mirrors) {
+ $_->{latitude} || $_->{longitude} or next;
+ my $PI = 3.14159265358979;
+ my $x = $latitude - $_->{latitude};
+ my $y = ($longitude - $_->{longitude}) * cos($_->{latitude} / 180 * $PI);
+ $_->{proximity} = sqrt($x * $x + $y * $y);
+ }
+ my ($best) = sort { $a->{proximity} <=> $b->{proximity} } @$mirrors;
+
+ foreach (@$mirrors) {
+ $_->{proximity_corrected} = $_->{proximity} * _random_correction();
+ $_->{proximity_corrected} *= _between_country_correction($country_code, $_->{country}) if $best;
+ $_->{proximity_corrected} *= _between_continent_correction($best->{continent}, $_->{continent}) if $best;
+ }
+ @$mirrors = sort { $a->{proximity_corrected} <=> $b->{proximity_corrected} } @$mirrors;
+}
+
+# add +/- 5% random
+sub _random_correction() {
+ my $correction = 0.05;
+ 1 + (rand() - 0.5) * $correction * 2;
+}
+
+sub _between_country_correction {
+ my ($here, $mirror) = @_;
+ $here && $mirror or return 1;
+ $here eq $mirror ? 0.5 : 1;
+}
+sub _between_continent_correction {
+ my ($here, $mirror) = @_;
+ $here && $mirror or return 1;
+ $here eq $mirror ? 0.5 : # favor same continent
+ $here eq 'SA' && $mirror eq 'NA' ? 0.9 : # favor going "South America" -> "North America"
+ 1;
+}
+
+sub _mirrors_raw {
+ my ($urpm, $url) = @_;
+
+ $urpm->{log}(N("getting mirror list from %s", $url));
+ my @l = urpm::download::get_content($urpm, $url) or die "mirror list not found";
+ @l;
+}
+
+sub _mirrors_filtered {
+ my ($urpm, $mirrorlist) = @_;
+
+ grep {
+ $_->{type} eq 'distrib'; # type=updates seems to be history, and type=iso is not interesting here
+ } map { chomp; parse_LDAP_namespace_structure($_) } _mirrors_raw($urpm, $mirrorlist);
+}
+
+sub _MIRRORLIST() {
+ my $product_id = parse_LDAP_namespace_structure(cat_('/etc/product.id'));
+ _mandriva_mirrorlist($product_id);
+}
+sub _mandriva_mirrorlist {
+ my ($product_id, $o_arch) = @_;
+
+ #- contact the following URL to retrieve the list of mirrors.
+ #- http://wiki.mandriva.com/en/Product_id
+ my $product_type = lc($product_id->{type}); $product_id =~ s/\s//g;
+ my $arch = $o_arch || $product_id->{arch};
+
+ "http://api.mandriva.com/mirrors/$product_type.$product_id->{version}.$arch.list";
+}
+
+sub parse_LDAP_namespace_structure {
+ my ($s) = @_;
+ my %h = map { /(.*?)=(.*)/ ? ($1 => $2) : () } split(',', $s);
+ \%h;
+}
+
+1;