diff options
Diffstat (limited to 'urpm')
| -rw-r--r-- | urpm/args.pm | 1 | ||||
| -rw-r--r-- | urpm/cfg.pm | 6 | ||||
| -rw-r--r-- | urpm/media.pm | 85 | ||||
| -rw-r--r-- | urpm/mirrors.pm | 212 |
4 files changed, 288 insertions, 16 deletions
diff --git a/urpm/args.pm b/urpm/args.pm index d809ee9d..73665d9d 100644 --- a/urpm/args.pm +++ b/urpm/args.pm @@ -316,6 +316,7 @@ my %options_spec = ( 'xml-info=s' => \$options{'xml-info'}, 'no-probe' => sub { $options{probe_with} = undef }, distrib => sub { $options{distrib} = 1 }, + 'mirrorlist=s' => \$options{mirrorlist}, interactive => sub { $options{interactive} = 1 }, 'all-media' => sub { $options{allmedia} = 1 }, 'from=s' => \$options{mirrors_url}, diff --git a/urpm/cfg.pm b/urpm/cfg.pm index 03175d5f..3283c6e7 100644 --- a/urpm/cfg.pm +++ b/urpm/cfg.pm @@ -140,11 +140,13 @@ sub load_config_raw { $err = N("medium `%s' is defined twice, aborting", $name); return; } - $block = { name => $name, url => $url }; + $block = { name => $name, $url ? (url => $url) : () }; } elsif (/^(hdlist |list |with_hdlist |with_synthesis + |with-dir + |mirrorlist |media_info_dir |removable |md5sum @@ -294,7 +296,7 @@ sub write_ini_config { foreach (@$blocks) { my %h = %$_; my $section = delete $h{'with-dir'} || '_'; - $uniq{$section}++ or die "conflicting with-dir value\n"; + $uniq{$section}++ or die "conflicting with-dir value $section\n"; foreach (difference2([ $cfg->Parameters($section) ], [ keys %h ])) { # remove those options which are no more wanted diff --git a/urpm/media.pm b/urpm/media.pm index 2e91fdd5..c7995b52 100644 --- a/urpm/media.pm +++ b/urpm/media.pm @@ -16,6 +16,7 @@ our @PER_MEDIA_OPT = qw( key-ids list media_info_dir + mirrorlist name no-media-info noreconfigure @@ -56,6 +57,7 @@ sub _only_media_opts_write { my ($m) = @_; my $c = only_media_opts($m); delete $c->{media_info_dir} if $c->{media_info_dir} eq 'media_info'; + delete $c->{url} if $c->{mirrorlist}; $c; } @@ -84,7 +86,7 @@ sub read_config_add_passwords { my ($urpm, $config) = @_; my @netrc = read_private_netrc($urpm) or return; - foreach (@{$config->{media}}) { + foreach (grep { $_->{url} } @{$config->{media}}) { my $u = urpm::download::parse_url_with_login($_->{url}) or next; if (my ($e) = grep { ($_->{default} || $_->{machine} eq $u->{machine}) && $_->{login} eq $u->{login} } @netrc) { $_->{url} = sprintf('%s://%s:%s@%s%s', $u->{proto}, $u->{login}, $e->{password}, $u->{machine}, $u->{dir}); @@ -98,7 +100,7 @@ sub remove_passwords_and_write_private_netrc { my ($urpm, $config) = @_; my @l; - foreach (@{$config->{media}}) { + foreach (grep { $_->{url} } @{$config->{media}}) { my $u = urpm::download::parse_url_with_login($_->{url}) or next; #- check whether a password is visible $u->{password} or next; @@ -149,7 +151,7 @@ sub read_config { foreach my $m (@{$config->{media}}) { my $medium = _only_media_opts_read($m); - if (!$medium->{url}) { + if (!$medium->{url} && !$medium->{mirrorlist}) { #- recover the url the old deprecated way... #- only useful for migration, new urpmi.cfg will use netrc recover_url_from_list($urpm, $medium); @@ -180,7 +182,7 @@ sub check_existing_medium { my ($urpm, $medium) = @_; my $err; - if (!$medium->{url}) { + if (!$medium->{url} && !$medium->{mirrorlist}) { $err = $medium->{virtual} ? N("virtual medium \"%s\" should have a clear url, medium ignored", $medium->{name}) : @@ -602,6 +604,8 @@ sub _parse_media { delete @$_{qw(start end)}; _parse_synthesis_or_ignore($urpm, $_, $options->{callback}); + _pick_mirror_if_needed($urpm, $_, ''); + if ($_->{searchmedia}) { $urpm->{searchmedia} = 1; $urpm->{log}(N("Search start: %s end: %s", $_->{start}, $_->{end})); @@ -644,7 +648,7 @@ sub _compute_flags_for_instlist { #- add a new medium, sync the config file accordingly. #- returns the new medium's name. (might be different from the requested #- name if index_name was specified) -#- options: ignore, index_name, nolock, update, virtual, media_info_dir, xml-info +#- options: ignore, index_name, nolock, update, virtual, media_info_dir, mirrorlist, with-dir, xml-info sub add_medium { my ($urpm, $name, $url, $with_synthesis, %options) = @_; @@ -670,7 +674,7 @@ sub add_medium { url => $url, modified => !$options{ignore}, }; - foreach (qw(downloader update ignore media_info_dir xml-info)) { + foreach (qw(downloader update ignore media_info_dir mirrorlist with-dir xml-info)) { $medium->{$_} = $options{$_} if exists $options{$_}; } @@ -683,6 +687,17 @@ sub add_medium { probe_removable_device($urpm, $medium); } + if (!$medium->{url} && $options{mirrorlist}) { + # forcing the standard media_info_dir if undefined + $medium->{media_info_dir} ||= 'media_info'; + + require urpm::mirrors; + urpm::mirrors::try($urpm, $medium, sub { + # this is a little ugly since MD5SUM will be downloaded again later, but it's small enough... + _download_MD5SUM($urpm, $medium); + }) or return; + } + if ($with_synthesis) { _migrate__with_synthesis($medium, $with_synthesis); } elsif (!$medium->{media_info_dir}) { @@ -719,6 +734,7 @@ sub add_medium { #- - probe_with : force use of rpms instead of using synthesis #- - ask_media : callback to know whether each media should be added #- - only_updates : only add "update" media (used by rpmdrake) +#- - mirrorlist #- other options are passed to add_medium(): ignore, nolock, virtual sub add_distrib_media { my ($urpm, $name, $url, %options) = @_; @@ -728,7 +744,7 @@ sub add_distrib_media { my $distribconf; - if (my $dir = file_from_local_url($url)) { + if (my $dir = $url && file_from_local_url($url)) { urpm::removable::try_mounting($urpm, $dir) or $urpm->{error}(N("unable to mount the distribution medium")), return (); $distribconf = MDV::Distribconf->new($dir, undef); @@ -737,7 +753,15 @@ sub add_distrib_media { } else { unlink "$urpm->{cachedir}/partial/media.cfg"; - $distribconf = _new_distribconf_and_download($urpm, $url); + if ($options{mirrorlist}) { + $url and die "unexpected url $url together with mirrorlist $options{mirrorlist}\n"; + } + + my $m = { mirrorlist => $options{mirrorlist}, url => $url }; + try__maybe_mirrorlist($urpm, $m, sub { + $distribconf = _new_distribconf_and_download($urpm, $m->{url}); + }); + $url = $m->{url}; if ($distribconf) { $distribconf->parse_mediacfg("$urpm->{cachedir}/partial/media.cfg") @@ -795,6 +819,7 @@ sub add_distrib_media { !$use_copied_synthesis && $options{probe_with} ? ($options{probe_with} => 1) : (), index_name => $name ? undef : 0, $add_by_default ? () : (ignore => 1), + $options{mirrorlist} ? ('with-dir' => $distribconf->getpath($media, 'path')) : (), %options, # the following override %options update => $is_update_media ? 1 : undef, @@ -900,6 +925,8 @@ sub _clean_statedir_medium_files { sub _probe_with_try_list { my ($urpm, $medium, $f) = @_; + $medium->{mirrorlist} and die "_probe_with_try_list does not handle mirrorlist\n"; + my @media_info_dirs = ('media_info', '.'); my $base = file_from_local_url($medium->{url}) || $medium->{url}; @@ -919,6 +946,8 @@ sub _probe_with_try_list { sub may_reconfig_urpmi { my ($urpm, $medium) = @_; + $medium->{url} or return; # we should handle mirrorlist? + my $f; if (my $dir = file_from_local_url($medium->{url})) { $f = reduce_pathname("$dir/reconfig.urpmi"); @@ -1116,6 +1145,7 @@ sub get_descriptions_local { $medium->{ignore} = 1; } } +#- not handling different mirrors since the file is not always available sub get_descriptions_remote { my ($urpm, $medium) = @_; @@ -1158,15 +1188,16 @@ sub get_synthesis__local { sub get_synthesis__remote { my ($urpm, $medium, $callback, $quiet) = @_; - if (urpm::download::sync($urpm, $medium, [ _url_with_synthesis($medium) ], + my $ok = try__maybe_mirrorlist($urpm, $medium, sub { + urpm::download::sync($urpm, $medium, [ _url_with_synthesis($medium) ], quiet => $quiet, callback => $callback) && - file_size(cachedir_with_synthesis($urpm, $medium)) >= 20) { - 1; - } else { + file_size(cachedir_with_synthesis($urpm, $medium)) >= 20; + }); + if (!$ok) { chomp(my $err = $@); $urpm->{error}(N("...retrieving failed: %s", $err)); - 0; } + $ok; } #- check copied/downloaded file has right signature. @@ -1425,6 +1456,8 @@ sub _update_medium_ { unlink "$urpm->{cachedir}/partial/$_"; } + _pick_mirror_if_needed($urpm, $medium, 'allow-cache-update'); + #- check for a reconfig.urpmi file (if not already reconfigured) if (!$medium->{noreconfigure}) { may_reconfig_urpmi($urpm, $medium); @@ -1503,7 +1536,7 @@ sub _update_media__handle_some_flags { $medium->{modified} = 0; } elsif ($all) { #- if we're rebuilding all media, mark them as modified (except removable ones) - $medium->{modified} ||= $medium->{url} !~ m!^removable!; + $medium->{modified} ||= !($medium->{url} && $medium->{url} =~ m!^removable!); } } } @@ -1625,6 +1658,30 @@ sub _any_media_info__or_download { $f; } +#- side-effects: +#- + those of urpm::mirrors::pick_one ($urpm->{mirrors_cache}, $medium->{url}) +sub _pick_mirror_if_needed { + my ($urpm, $medium, $allow_cache_update) = @_; + + $medium->{mirrorlist} && !$medium->{url} or return; + + require urpm::mirrors; + urpm::mirrors::pick_one($urpm, $medium, $allow_cache_update); +} + +#- side-effects: +#- + those of urpm::mirrors::try ($urpm->{mirrors_cache}, $medium->{url}) +sub try__maybe_mirrorlist { + my ($urpm, $medium, $try) = @_; + + if ($medium->{mirrorlist}) { + require urpm::mirrors; + urpm::mirrors::try($urpm, $medium, $try); + } else { + $try->(); + } +} + #- clean params and depslist computation zone. sub clean { my ($urpm) = @_; diff --git a/urpm/mirrors.pm b/urpm/mirrors.pm new file mode 100644 index 00000000..17366b38 --- /dev/null +++ b/urpm/mirrors.pm @@ -0,0 +1,212 @@ +package urpm::mirrors; + +# $Id: $ + +use strict; +use urpm::util; +use urpm::msg; +use urpm::download; + + +#- $medium fields used: mirrorlist, with-dir +#- side-effects: $medium->{url} +#- + those of _pick_one ($urpm->{mirrors_cache}) +sub try { + my ($urpm, $medium, $try) = @_; + + for (my $nb = 1; $nb < $urpm->{options}{'max-round-robin-tries'}; $nb++) { + my $url = _pick_one($urpm, $medium->{mirrorlist}, $nb == 1, '') or return; + $urpm->{info}(N("trying again with mirror %s", $url)) if $nb > 1; + $medium->{url} = _add__with_dir($url, $medium->{'with-dir'}); + $try->() and return 1; + black_list($urpm, $medium->{mirrorlist}, $url); + } + 0; +} + +#- side-effects: none +sub _add__with_dir { + my ($url, $with_dir) = @_; + reduce_pathname($url . ($with_dir ? "/$with_dir" : '')); +} + +#- side-effects: $medium->{url} +#- + those of _pick_one ($urpm->{mirrors_cache}) +sub pick_one { + my ($urpm, $medium, $allow_cache_update) = @_; + + my $url = _pick_one($urpm, $medium->{mirrorlist}, 'must_succeed', $allow_cache_update); + $medium->{url} = _add__with_dir($url, $medium->{'with-dir'}); +} + +#- side-effects: $urpm->{mirrors_cache} +sub _pick_one { + my ($urpm, $mirrorlist, $must_succeed, $allow_cache_update) = @_; + my $cache = _cache($urpm, $mirrorlist); + + if ($allow_cache_update && $cache->{time} && + time() > $cache->{time} + 24*60*60 * $urpm->{options}{'days-between-mirrorlist-update'}) { + $urpm->{log}("not using outdated cached mirror list"); + %$cache = (); + } + + if (!$cache->{chosen}) { + if (!$cache->{list}) { + $cache->{list} = [ _list($urpm, $mirrorlist) ]; + $cache->{time} = time(); + } + + $cache->{chosen} = $cache->{list}[0]{url} or do { + $must_succeed and $urpm->{fatal}(10, N("Could not find a mirror from mirrorlist %s", $mirrorlist)); + return; + }; + _save_cache($urpm); + } + if ($cache->{nb_uses}++) { + $urpm->{debug} and $urpm->{debug}("using mirror $cache->{chosen}"); + } else { + $urpm->{log}("using mirror $cache->{chosen}"); + } + + $cache->{chosen}; +} +#- side-effects: $urpm->{mirrors_cache} +sub black_list { + my ($urpm, $mirrorlist, $url) = @_; + my $cache = _cache($urpm, $mirrorlist); + + @{$cache->{list}} = grep { $_->{url} ne $url } @{$cache->{list}}; + delete $cache->{chosen}; +} +#- side-effects: $urpm->{mirrors_cache} +sub _cache { + my ($urpm, $mirrorlist) = @_; + my $full_cache = $urpm->{mirrors_cache} ||= _load_cache($urpm); + $full_cache->{$mirrorlist} ||= {}; +} +sub cache_file { + my ($urpm) = @_; + my $cache_file = "$urpm->{cachedir}/mirrors.cache"; +} +sub _load_cache { + my ($urpm) = @_; + my $cache; + if (-e cache_file($urpm)) { + $urpm->{debug} and $urpm->{debug}("loading mirrors cache"); + $cache = eval(cat_(cache_file($urpm))); + $@ and $urpm->{error}("failed to read " . cache_file($urpm) . ": $@"); + $_->{nb_uses} = 0 foreach values %$cache; + } + $cache || {}; +} +sub _save_cache { + my ($urpm) = @_; + require Data::Dumper; + my $s = Data::Dumper::Dumper($urpm->{mirrors_cache}); + $s =~ s/.*?=//; # get rid of $VAR1 = + output_safe(cache_file($urpm), $s); +} + +#- side-effects: none +sub _list { + my ($urpm, $mirrorlist) = @_; + + # expand the variable + $mirrorlist = _MIRRORLIST() if $mirrorlist eq '$MIRRORLIST'; + + my @mirrors = _mirrors_filtered($urpm, $mirrorlist); + add_proximity_and_sort($urpm, \@mirrors); + @mirrors; +} + +#- side-effects: $mirrors +sub add_proximity_and_sort { + my ($urpm, $mirrors) = @_; + + my ($latitude, $longitude, $country_code); + + require Time::ZoneInfo; + if (my $zone = Time::ZoneInfo->current_zone) { + if (my $zones = Time::ZoneInfo->new) { + if (($latitude, $longitude) = $zones->latitude_longitude_decimal($zone)) { + $country_code = $zones->country($zone); + $urpm->{log}(N("found geolocalisation %s %.2f %.2f from timezone %s", $country_code, $latitude, $longitude, $zone)); + } + } + } + defined $latitude && defined $longitude or return; + + foreach (@$mirrors) { + $_->{latitude} || $_->{longitude} or next; + my $PI = 3.14159265358979; + my $x = $latitude - $_->{latitude}; + my $y = ($longitude - $_->{longitude}) * cos($_->{latitude} / 180 * $PI); + $_->{proximity} = sqrt($x * $x + $y * $y); + } + my ($best) = sort { $a->{proximity} <=> $b->{proximity} } @$mirrors; + + foreach (@$mirrors) { + $_->{proximity_corrected} = $_->{proximity} * _random_correction(); + $_->{proximity_corrected} *= _between_country_correction($country_code, $_->{country}) if $best; + $_->{proximity_corrected} *= _between_continent_correction($best->{continent}, $_->{continent}) if $best; + } + @$mirrors = sort { $a->{proximity_corrected} <=> $b->{proximity_corrected} } @$mirrors; +} + +# add +/- 5% random +sub _random_correction() { + my $correction = 0.05; + 1 + (rand() - 0.5) * $correction * 2; +} + +sub _between_country_correction { + my ($here, $mirror) = @_; + $here && $mirror or return 1; + $here eq $mirror ? 0.5 : 1; +} +sub _between_continent_correction { + my ($here, $mirror) = @_; + $here && $mirror or return 1; + $here eq $mirror ? 0.5 : # favor same continent + $here eq 'SA' && $mirror eq 'NA' ? 0.9 : # favor going "South America" -> "North America" + 1; +} + +sub _mirrors_raw { + my ($urpm, $url) = @_; + + $urpm->{log}(N("getting mirror list from %s", $url)); + my @l = urpm::download::get_content($urpm, $url) or die "mirror list not found"; + @l; +} + +sub _mirrors_filtered { + my ($urpm, $mirrorlist) = @_; + + grep { + $_->{type} eq 'distrib'; # type=updates seems to be history, and type=iso is not interesting here + } map { chomp; parse_LDAP_namespace_structure($_) } _mirrors_raw($urpm, $mirrorlist); +} + +sub _MIRRORLIST() { + my $product_id = parse_LDAP_namespace_structure(cat_('/etc/product.id')); + _mandriva_mirrorlist($product_id); +} +sub _mandriva_mirrorlist { + my ($product_id, $o_arch) = @_; + + #- contact the following URL to retrieve the list of mirrors. + #- http://wiki.mandriva.com/en/Product_id + my $product_type = lc($product_id->{type}); $product_id =~ s/\s//g; + my $arch = $o_arch || $product_id->{arch}; + + "http://api.mandriva.com/mirrors/$product_type.$product_id->{version}.$arch.list"; +} + +sub parse_LDAP_namespace_structure { + my ($s) = @_; + my %h = map { /(.*?)=(.*)/ ? ($1 => $2) : () } split(',', $s); + \%h; +} + +1; |