path: root/urpm/media.pm

package urpm::media;

# $Id$

use strict;
use urpm 'file_from_local_medium', 'is_local_medium';
use urpm::msg;
use urpm::util;
use urpm::removable;
use urpm::lock;
use urpm::md5sum;
use MDV::Distribconf;


our @PER_MEDIA_OPT = qw(
    disable-certificate-check
    downloader
    ignore
    key-ids
    list
    media_info_dir
    mirrorlist
    name
    no-media-info
    noreconfigure
    priority-upgrade
    removable
    static
    synthesis
    update
    url
    verify-rpm
    virtual
    with-dir
    with_hdlist
    with_synthesis
    xml-info
);

my @xml_media_info = ('info', 'files', 'changelog');

my @media_info_prefix_suffix = (
    [ 'synthesis.hdlist', '.cz' ],
    [ 'hdlist', '.cz' ],
    [ 'descriptions', '' ],
    [ 'names', '' ],
    [ 'MD5SUM', '' ],
    (map { [ $_, '.xml.lzma' ] } @xml_media_info),
);


sub get_medium_option {
    my ($urpm, $medium, $option_name) = @_;

    defined $medium->{$option_name} ? $medium->{$option_name} : $urpm->{options}{$option_name};
}

sub _is_iso {
    my ($removable_dev) = @_;
    $removable_dev && $removable_dev =~ /\.iso$/i;
}

sub only_media_opts {
    my ($m) = @_;
    my %m = map { $_ => $m->{$_} } grep { defined $m->{$_} } @PER_MEDIA_OPT;
    \%m;
}
sub _only_media_opts_read {
    my ($m) = @_;
    my $c = only_media_opts($m);
    $c->{media_info_dir} ||= 'media_info';
    $c->{iso} = delete $c->{removable} if $c->{removable} && _is_iso($c->{removable});
    $c;
}
sub _only_media_opts_write {
    my ($m) = @_;
    my $c = only_media_opts($m);
    delete $c->{media_info_dir} if $c->{media_info_dir} eq 'media_info';
    delete $c->{url} if $c->{mirrorlist};
    $c->{removable} = delete $c->{iso} if $c->{iso};
    $c;
}

sub read_private_netrc {
    my ($urpm) = @_;

    my @words = split(/\s+/, scalar cat_($urpm->{private_netrc}));
    my @l;
    my $e;
    while (@words) {
	my $keyword = shift @words;
	if ($keyword eq 'machine') {
	    push @l, $e = { machine => shift(@words) };
	} elsif ($keyword eq 'default') {
	    push @l, $e = { default => '' };
	} elsif ($keyword eq 'login' || $keyword eq 'password' || $keyword eq 'account') {
	    $e->{$keyword} = shift(@words);
	} else {
	    $urpm->{error}("unknown netrc command $keyword");
	}
    }
    @l;
}

sub read_config_add_passwords {
    my ($urpm, $config) = @_;

    my @netrc = read_private_netrc($urpm) or return;
    foreach (grep { $_->{url} } @{$config->{media}}) {
	my $u = urpm::download::parse_url_with_login($_->{url}) or next;
	if (my ($e) = grep { ($_->{default} || $_->{machine} eq $u->{machine}) && $_->{login} eq $u->{login} } @netrc) {
	    $_->{url} = sprintf('%s://%s:%s@%s%s', $u->{proto}, $u->{login}, $e->{password}, $u->{machine}, $u->{dir});
	} else {
	    $urpm->{log}(sprintf('no password found for %s@%s', $u->{login}, $u->{machine}));
	}
    }
}

sub remove_passwords_and_write_private_netrc {
    my ($urpm, $config) = @_;

    my @l;
    foreach (grep { $_->{url} } @{$config->{media}}) {
	my $u = urpm::download::parse_url_with_login($_->{url}) or next;
	#- check whether a password is visible
	$u->{password} or next;

	push @l, $u;
	$_->{url} = sprintf('%s://%s@%s%s', $u->{proto}, $u->{login}, $u->{machine}, $u->{dir});
    }
    {
	my $fh = urpm::sys::open_safe($urpm, '>', $urpm->{private_netrc}) or return;
	foreach my $u (@l) {
	    printf $fh "machine %s login %s password %s\n", $u->{machine}, $u->{login}, $u->{password};
	}
    }
    chmod 0600, $urpm->{private_netrc};
}

#- handle deprecated way of saving passwords
sub recover_url_from_list {
    my ($urpm, $medium) = @_;

    my $list = delete $medium->{list} or return;
    my $statedir_list = "$urpm->{statedir}/$list";

    #- /./ is end of url marker in list file (typically generated by a
    #- find . -name "*.rpm" > list
    #- for exportable list file.
    if (my @probe = map { m!^(.*)/\./! || m!^(.*)/[^/]*$! } cat_($statedir_list)) {
	$urpm->{log}("recovering url from $statedir_list");
	($medium->{url}) = sort { length($a) <=> length($b) } @probe;
	$urpm->{modified} = 1; #- ensure urpmi.cfg is handled using only partially hidden url + netrc, since file list won't be generated anymore
	unlink $statedir_list;
    }
}

sub _read_config__read_media_info {
    my ($urpm) = @_;

    require File::Glob;
    # we can't use perl's glob() because there could be spaces in 
    # $urpm->{mediacfgdir}
    my %url2mediamap;
    my %mirrorlist2mediamap;
    foreach my $media_dir (File::Glob::bsd_glob("$urpm->{mediacfgdir}/*")) {
	next if !-d $media_dir;

	$urpm->{debug} and $urpm->{debug}("parsing: $media_dir");

	my $media_cfg = $media_dir . '/media.cfg';
	my $distribconf = MDV::Distribconf->new($media_cfg, undef) or next;
	$distribconf->settree('mandriva');
	$distribconf->parse_mediacfg($media_cfg) or next;
    
	if (open(my $URLS, '<', $media_dir . '/url')) {
	    local $_;
	    while (<$URLS>) {
		chomp($_);
		foreach my $medium ($distribconf->listmedia) {
		    my $medium_path = reduce_pathname($_ . '/' . $distribconf->getpath($medium, 'path'));
		    $url2mediamap{$medium_path} = [$distribconf, $medium];
		}
	    }
	}

	if (open(my $MIRRORLISTS, '<', $media_dir . '/mirrorlist')) {
	    local $_;
	    while (<$MIRRORLISTS>) {
		my $mirrorlist = $_;
		chomp($mirrorlist);
		foreach my $medium ($distribconf->listmedia) {
		    my $medium_path = $distribconf->getpath($medium, 'path');
		    $mirrorlist2mediamap{$mirrorlist}{$medium_path} = [ $distribconf, $medium ];
		}
	    }
	}
    }
    (\%url2mediamap, \%mirrorlist2mediamap);
}

sub _associate_media_with_mediacfg {
    my ($urpm, $media) = @_;

    my ($url2mediamap, $mirrorlist2mediamap) = _read_config__read_media_info($urpm);
    foreach my $medium (@$media) {
	if ($medium->{mirrorlist}) {
	    $medium->{mediacfg} = $mirrorlist2mediamap->{$medium->{mirrorlist}}{$medium->{'with-dir'}};
	} elsif ($medium->{url}) {
	    $medium->{mediacfg} = $url2mediamap->{$medium->{url}};
	}
    }
}

#- Loads /etc/urpmi/urpmi.cfg and performs basic checks.
#- Does not handle old format: <name> <url> [with <path_hdlist>]
sub read_config {
    my ($urpm, $nocheck) = @_;
    return if $urpm->{media}; #- media already loaded
    $urpm->{media} = [];
    my $config = urpm::cfg::load_config($urpm->{config})
	or $urpm->{fatal}(6, $urpm::cfg::err);

    #- per-media options
    read_config_add_passwords($urpm, $config);

    my @media;
    foreach my $m (@{$config->{media}}) {
	my $medium = _only_media_opts_read($m);

	if (!$medium->{url} && !$medium->{mirrorlist}) {
	    #- recover the url the old deprecated way...
	    #- only useful for migration, new urpmi.cfg will use netrc
	    recover_url_from_list($urpm, $medium);
	    $medium->{url} or $urpm->{error}("unable to find url in list file $medium->{name}, medium ignored");
	}
	push @media, $medium;
    }

    # associate medias read from the config file with their description in a
    # media.cfg file
    # @media content will be modified and then add_existing medium will take 
    # care of copying the media to $urpm
    _associate_media_with_mediacfg($urpm, \@media);

    add_existing_medium($urpm, $_, $nocheck) foreach @media;

    eval { require urpm::ldap; urpm::ldap::load_ldap_media($urpm) };
}

#- if invalid, set {ignore}
sub check_existing_medium {
    my ($urpm, $medium) = @_;

    my $err;
    if (!$medium->{url} && !$medium->{mirrorlist}) { 
	$err = $medium->{virtual} ?
	  N("virtual medium \"%s\" should have a clear url, medium ignored",
			   $medium->{name}) :
	  N("unable to access list file of \"%s\", medium ignored", $medium->{name});
    } elsif (!$medium->{ignore} 
	     && !-r any_synthesis($urpm, $medium)) {
	$err = N("unable to access synthesis file of \"%s\", medium ignored", $medium->{name});
    }
    if ($err) {
	$medium->{ignore} = 1;
	$urpm->{error}($err);
    }
}

sub _migrate__with_synthesis {
    my ($medium, $with_synthesis) = @_;

    #- try to migrate to media_info_dir
    my $b = basename($with_synthesis);
    if ($b eq 'synthesis.hdlist.cz' || $b eq 'hdlist.cz') {
	$medium->{media_info_dir} = dirname($with_synthesis);
    } else {
	$with_synthesis =~ s/(synthesis\.)?(hdlist.*\.cz)$/synthesis.$2/;
	$medium->{with_synthesis} = $with_synthesis;
    }
}

#- probe medium to be used, take old medium into account too.
sub add_existing_medium {
    my ($urpm, $medium, $nocheck) = @_;

    if (name2medium($urpm, $medium->{name})) {
	$urpm->{error}(N("trying to override existing medium \"%s\", skipping", $medium->{name}));
	return;
    }

    if ($medium->{with_hdlist}) {
	_migrate__with_synthesis($medium, delete $medium->{with_hdlist});
	$urpm->{modified} = 1;
    }

    check_existing_medium($urpm, $medium) if !$nocheck;

    _migrate_removable_device($urpm, $medium);

    #- clear URLs for trailing /es.
    $medium->{url} and $medium->{url} =~ s|(.*?)/*$|$1|;

    push @{$urpm->{media}}, $medium;
}

sub file_from_file_url {
    my ($url) = @_;
    $url =~ m!^(?:file:/)?(/.*)! && $1;
}

sub _local_file {
    my ($medium) = @_;
    $medium->{url} && file_from_file_url($medium->{url});
}

sub _is_local_virtual {
    my ($medium) = @_;
    $medium->{virtual} && _local_file($medium);
}
sub _is_remote_virtual {
    my ($medium) = @_;
    $medium->{virtual} && !_local_file($medium);
}

sub _url_with_synthesis_basename {
    my ($medium) = @_;

    $medium->{with_synthesis}
      ? basename($medium->{with_synthesis})
      : 'synthesis.hdlist.cz';
}
sub _synthesis_dir_rel {
    my ($medium) = @_;
    $medium->{'no-media-info'} || $medium->{unknown_media_info} and return;

    $medium->{with_synthesis}
      ? "$medium->{with_synthesis}/.."
      : $medium->{media_info_dir};
}
sub _synthesis_dir {
    my ($medium) = @_;
    my $rel = _synthesis_dir_rel($medium) or return;

    my $base = file_from_local_medium($medium) || $medium->{url};
    reduce_pathname("$base/$rel");
}

# the difference between _valid_synthesis_dir and _synthesis_dir
# is only to handle upgrades from older urpmi.cfg where no {media_info_dir}
# meant no-media-info
sub _valid_synthesis_dir {
    my ($medium) = @_;
    my $dir = _synthesis_dir($medium);
    $dir && -d $dir;
}

sub _url_with_synthesis_rel {
    my ($medium) = @_;

    $medium->{with_synthesis} ||
      "$medium->{media_info_dir}/synthesis.hdlist.cz";

}
sub _url_with_synthesis {
    my ($medium) = @_;
    my $rel = _url_with_synthesis_rel($medium) or return;

    my $base = file_from_local_medium($medium) || $medium->{url};
    reduce_pathname("$base/$rel");
}

sub synthesis {
    my ($medium) = @_;
    statedir_media_info_basename($medium, 'synthesis.hdlist', '.cz');
}

sub statedir_media_info_basename {
    my ($medium, $prefix, $suffix) = @_;
    $medium->{name} && "$prefix.$medium->{name}$suffix";
}

sub statedir_media_info_dir {
    my ($urpm, $medium) = @_;
    $medium->{name} && "$urpm->{statedir}/$medium->{name}";
}

sub old_statedir_media_info_file {
    my ($urpm, $medium, $prefix, $suffix) = @_;
    $medium->{name} && "$urpm->{statedir}/" . statedir_media_info_basename($medium, $prefix, $suffix);
}
sub new_statedir_media_info_file {
    my ($urpm, $medium, $prefix, $suffix) = @_;
    my $dir = statedir_media_info_dir($urpm, $medium);
    $dir && "$dir/$prefix$suffix";
}
sub statedir_media_info_file {
    my ($urpm, $medium, $prefix, $suffix) = @_;
    my $dir = statedir_media_info_dir($urpm, $medium);
    -d $dir ? "$dir/$prefix$suffix" : old_statedir_media_info_file($urpm, $medium, $prefix, $suffix);
}
sub statedir_synthesis {
    my ($urpm, $medium) = @_;
    statedir_media_info_file($urpm, $medium, 'synthesis.hdlist', '.cz');
}
sub statedir_descriptions {
    my ($urpm, $medium) = @_;
    statedir_media_info_file($urpm, $medium, 'descriptions', '');
}
sub statedir_names {
    my ($urpm, $medium) = @_;
    old_statedir_media_info_file($urpm, $medium, 'names', '');
}
sub statedir_MD5SUM {
    my ($urpm, $medium) = @_;
    statedir_media_info_file($urpm, $medium, 'MD5SUM', '');
}
sub statedir_hdlist {
    my ($urpm, $medium) = @_;
    statedir_media_info_file($urpm, $medium, 'hdlist', '.cz');
}
sub statedir_xml_info {
    my ($urpm, $medium, $xml_info) = @_;
    statedir_media_info_file($urpm, $medium, $xml_info, '.xml.lzma');
}
sub cachedir_with_synthesis {
    my ($urpm, $medium) = @_;
    _url_with_synthesis($medium) && "$urpm->{cachedir}/partial/synthesis.hdlist.cz";
}
sub any_synthesis {
    my ($urpm, $medium) = @_;
    my $f = _is_local_virtual($medium) ? _url_with_synthesis($medium)
      : statedir_synthesis($urpm, $medium);
    -e $f && $f;
}
sub any_media_info_file {
    my ($urpm, $medium, $prefix, $suffix, $quiet, $o_callback) = @_;

    if (my $base = _local_file($medium)) {
	my $f = $medium->{with_synthesis}
	  ? reduce_pathname("$base/$prefix." . _synthesis_suffix($medium) . $suffix)
	  : _synthesis_dir($medium) . "/$prefix$suffix";

	if (! -e $f) {
	    # in some weird cases (iso on disk), the hdlist is not available where it should be,
	    # but we can use the statedir copy
	    $f = statedir_media_info_file($urpm, $medium, $prefix, $suffix);
	}

	-e $f && $f;
    } else {
	_any_media_info__or_download($urpm, $medium, $prefix, $suffix, $quiet, $o_callback);
    }
}
sub any_hdlist {
    my ($urpm, $medium, $quiet) = @_;
    any_media_info_file($urpm, $medium, 'hdlist', '.cz', $quiet, \&urpm::download::sync_logger);
}
sub any_xml_info {
    my ($urpm, $medium, $xml_info, $quiet, $o_callback) = @_;
    any_media_info_file($urpm, $medium, $xml_info, '.xml.lzma', $quiet, $o_callback || \&urpm::download::sync_logger);
}

sub name2medium {
    my ($urpm, $name) = @_;
    my ($medium) = grep { $_->{name} eq $name } @{$urpm->{media}};
    $medium;
}

sub userdirs {
    my ($urpm) = @_;
    my $prefix = urpm::userdir_prefix($urpm);
    grep { m!^\Q$prefix\E\d+$! && -d $_ && ! -l $_ } glob("$prefix*");
}

sub remove_user_media_info_files {
    my ($urpm, $medium) = @_;

    foreach my $dir (userdirs($urpm)) {
	require File::Glob;
	# we can't use perl's glob() because $medium->{name} can contain spaces
	my @files = map { File::Glob::bsd_glob("$dir/*.$medium->{name}.$_") } 'cz', 'xml.lzma' or next;

	$urpm->{log}("cleaning $dir");
	foreach (@files) {
	    unlink $_ or $urpm->{error}("removing $_ failed");
	}
    }
}

#- probe device associated with a removable device.
sub _migrate_removable_device {
    my ($urpm, $medium) = @_;

    $medium->{url} or return;

    # always drop {removable}, it is obsolete
    # (nb: for iso files, {removable} has already been renamed into {iso} internally)
    delete $medium->{removable};

    if (my $url = _migrate_removable_url($medium->{url})) {
	$medium->{url} = $url;
    } else {
	$urpm->{error}(N("failed to migrate removable device, ignoring media"));
	$medium->{ignore} = 1;
    }
}

sub _migrate_removable_url {
    my ($url) = @_;

    if ($url =~ /^removable/) {
	$url =~ s!^removable(.*?)://!/!;
	if ($url =~ s!/(mnt|media)/cd\w+/?!cdrom://!i) {
	    # success!
	} else {
	    return;
	}
    }
    $url;
}


#- Writes the urpmi.cfg file.
sub write_urpmi_cfg {
    my ($urpm) = @_;

    #- avoid trashing exiting configuration if it wasn't loaded
    $urpm->{media} or return;

    my $config = {
	#- global config options found in the config file, without the ones
	#- set from the command-line
	global => $urpm->{global_config},
	media => [ map { _only_media_opts_write($_) } grep { !$_->{external} } @{$urpm->{media}} ],
    };
    remove_passwords_and_write_private_netrc($urpm, $config);

    urpm::cfg::dump_config($urpm->{config}, $config)
	or $urpm->{fatal}(6, N("unable to write config file [%s]", $urpm->{config}));

    $urpm->{log}(N("wrote config file [%s]", $urpm->{config}));

    #- everything should be synced now.
    delete $urpm->{modified};
}

sub write_config {
    my ($urpm) = @_;

    write_urpmi_cfg($urpm);
}


sub _tempignore {
    my ($medium, $ignore) = @_;
    $medium->{ignore} = $ignore;
}

#- read urpmi.cfg file as well as necessary synthesis files
#- options :
#-	root (deprecated, set directly $urpm->{root})
#-	cmdline_skiplist
#-      download_callback (used by _auto_update_media)
#-
#-	callback (urpmf)
#-	nodepslist (for urpmq, urpmf: when we don't need the synthesis)
#-	no_skiplist (urpmf)
#-
#-	synthesis (use this synthesis file, and only this synthesis file)
#-
#-	parallel
#-	usedistrib (otherwise uses urpmi.cfg)
#-	  media
#-	  excludemedia
#-	  sortmedia
#-
#-	  update
#-	  searchmedia
sub configure {
    my ($urpm, %options) = @_;

    clean($urpm);

    $options{parallel} && $options{usedistrib} and $urpm->{fatal}(1, N("Can't use parallel mode with use-distrib mode"));

    if ($options{parallel}) {
	require urpm::parallel;
	urpm::parallel::configure($urpm, $options{parallel});

	if (!$options{media} && $urpm->{parallel_handler}{media}) {
	    $options{media} = $urpm->{parallel_handler}{media};
	    $urpm->{log}->(N("using associated media for parallel mode: %s", $options{media}));
	}
    } else {
	#- nb: can't have both parallel and root
	$urpm->{root} = $options{root} if $options{root};
    }

    if ($urpm->{root} && ! -c "$urpm->{root}/dev/null") {
	mkdir "$urpm->{root}/dev";
	system("/bin/cp", "-a", '/dev/null', "$urpm->{root}/dev");
    }

    if ($options{synthesis}) {
	if ($options{synthesis} ne 'none') {
	    #- synthesis take precedence over media, update options.
	    $options{media} || $options{excludemedia} || $options{sortmedia} || $options{update} || $options{usedistrib} || $options{parallel} and
	      $urpm->{fatal}(1, N("--synthesis cannot be used with --media, --excludemedia, --sortmedia, --update, --use-distrib or --parallel"));

	    my $synthesis = $options{synthesis};
	    if ($synthesis !~ m!^/!) {
		require Cwd;
		$synthesis = Cwd::getcwd() . '/' . $synthesis;
	    }
	    my ($url, $with) = $synthesis =~ m!(.*)/+(media_info/+synthesis\.hdlist\.cz)$! ? ($1, $2) : 
	      (dirname($synthesis), basename($synthesis));

            $urpm->{media} = [];
	    add_medium($urpm, 'Virtual', $url, $with, %options, virtual => 1, on_the_fly => 1);
	}
    } elsif ($options{usedistrib}) {
            $urpm->{media} = [];
            add_distrib_media($urpm, "Virtual", $options{usedistrib}, %options, virtual => 1, on_the_fly => 1);
    } else {
	    read_config($urpm, '');
	    if (!$options{media} && $urpm->{options}{'default-media'}) {
		$options{media} = $urpm->{options}{'default-media'};
	    }
    }

	if ($options{media}) {
	    delete $_->{modified} foreach @{$urpm->{media} || []};
	    select_media($urpm, split /,/, $options{media});
	    foreach (@{$urpm->{media} || []}) {
		_tempignore($_, !$_->{modified});
	    }
	}
	if ($options{searchmedia}) {
	   foreach (select_media_by_name($urpm, [ split /,/, $options{searchmedia} ])) {
	       #- Ensure this media is selected
	       $_->{modified} = 1;
	       _tempignore($_, 0);
	       $_->{searchmedia} = 1;
	   }
	}
	if ($options{excludemedia}) {
	    delete $_->{modified} foreach @{$urpm->{media} || []};
	    foreach (select_media_by_name($urpm, [ split /,/, $options{excludemedia} ])) {
		$_->{modified} = 1;
		#- this is only a local ignore that will not be saved.
		_tempignore($_, 1);
	    }
	}
	if ($options{sortmedia}) {
	    my @sorted_media = map { select_media_by_name($urpm, [$_]) } split(/,/, $options{sortmedia});
	    my @remaining = difference2($urpm->{media}, \@sorted_media);
	    $urpm->{media} = [ @sorted_media, @remaining ];
	}
	_auto_update_media($urpm, %options);

	_pick_mirror_if_needed($urpm, $_, '') foreach non_ignored_media($urpm, $options{update});

	parse_media($urpm, \%options) if !$options{nodepslist};

    #- determine package to withdraw (from skip.list file) only if something should be withdrawn.
    if (!$options{nodepslist}) {
	_compute_flags_for_skiplist($urpm, $options{cmdline_skiplist}) if !$options{no_skiplist};
	_compute_flags_for_instlist($urpm);
    }
}

#- for remote "virtual" media
#- options: download_callback, nomd5sum, quiet, nopubkey
sub _auto_update_media {
    my ($urpm, %options) = @_;

    $options{callback} = delete $options{download_callback};

    foreach (grep { _is_remote_virtual($_) || $urpm->{options}{'auto-update'} } 
	       non_ignored_media($urpm, $options{update})) {
	_update_medium($urpm, $_, %options);
    }
}

sub non_ignored_media {
    my ($urpm, $b_only_marked_update) = @_;

    grep { !$_->{ignore} && (!$b_only_marked_update || $_->{update}) } @{$urpm->{media} || []};
}

sub all_media_to_update {
    my ($urpm, $b_only_marked_update) = @_;

    grep { !$_->{ignore}
	     && !$_->{static} && !urpm::is_cdrom_url($_->{url}) && !$_->{iso}
	     && (!$b_only_marked_update || $_->{update});
	} @{$urpm->{media} || []};
}

sub parse_media {
    my ($urpm, $options) = @_;

    foreach (non_ignored_media($urpm, $options->{update})) {
	delete @$_{qw(start end)};
	_parse_synthesis_or_ignore($urpm, $_, $options->{callback});

	if ($_->{searchmedia}) {
	    $urpm->{searchmedia} = 1;
	    $urpm->{debug} and $urpm->{debug}("Search start: %s end: %s", $_->{start}, $_->{end});
	}

        $< == 0 and _generate_medium_names($urpm, $_);
    }
}

sub _compute_flags_for_skiplist {
    my ($urpm, $cmdline_skiplist) = @_;
    my %uniq;
    $urpm->compute_flags(
	urpm::sys::get_packages_list($urpm->{skiplist}, $cmdline_skiplist),
	skip => 1,
	callback => sub {
	    my ($urpm, $pkg) = @_;
	    $pkg->is_arch_compat && ! exists $uniq{$pkg->fullname} or return;
	    $uniq{$pkg->fullname} = undef;
	    $urpm->{debug} and $urpm->{debug}(N("skipping package %s", scalar($pkg->fullname)));
	},
    );
}

sub _compute_flags_for_instlist {
    my ($urpm) = @_;

    my %uniq;
    $urpm->compute_flags(
	urpm::sys::get_packages_list($urpm->{instlist}),
	disable_obsolete => 1,
	callback => sub {
	    my ($urpm, $pkg) = @_;
	    $pkg->is_arch_compat && ! exists $uniq{$pkg->fullname} or return;
	    $uniq{$pkg->fullname} = undef;
	    $urpm->{log}(N("would install instead of upgrade package %s", scalar($pkg->fullname)));
	},
    );

}

#- add a new medium, sync the config file accordingly.
#- returns the new medium's name. (might be different from the requested
#- name if index_name was specified)
#- options: ignore, index_name, nolock, update, virtual, media_info_dir, mirrorlist, with-dir, xml-info, on_the_fly
sub add_medium {
    my ($urpm, $name, $url, $with_synthesis, %options) = @_;

    #- make sure configuration has been read.
    $urpm->{media} or die "caller should have used ->read_config or ->configure first";

    #- if a medium with that name has already been found, we have to exit now
    if (defined $options{index_name}) {
	my ($i, $basename) = ($options{index_name}, $name);
	while (1) {
	    ++$i;
	    $name = $basename . ($i == 1 ? '' : $i);
	    last if !name2medium($urpm, $name);
	}
    } else {
	name2medium($urpm, $name) and $urpm->{fatal}(5, N("medium \"%s\" already exists", $name));
    }

    $url =~ s,/*$,,; #- clear URLs for trailing /es.

    #- creating the medium info.
    my $medium = { name => $name, 
		url => $url, 
		modified => !$options{ignore}, 
	    };
    foreach (qw(downloader update ignore media_info_dir mirrorlist with-dir xml-info)) {
	$medium->{$_} = $options{$_} if exists $options{$_};
    }

    #- those files must not be there (cf mdvbz#36267)
    _clean_statedir_medium_files($urpm, $medium);
    if (!($options{virtual} && _local_file($medium))
	  && !$urpm->{urpmi_root}) { # with --urpmi-root, we do not use statedir_media_info_file to allow compatibility with older urpmi
	mkdir statedir_media_info_dir($urpm, $medium), 0755;
    }

    if ($options{virtual}) {
	$medium->{virtual} = 1;
    } else {
	_migrate_removable_device($urpm, $medium);
    }

    if ($with_synthesis) {
	_migrate__with_synthesis($medium, $with_synthesis);
    } elsif (!$medium->{media_info_dir}) {
	if (!is_local_medium($medium)) {
	    $medium->{media_info_dir} = 'media_info';
	} else {
	    $medium->{unknown_media_info} = 1;
	}
    }

    #- local media have priority, other are added at the end.
    my $inserted;
    my $ignore_text = $medium->{ignore} ? ' ' . N("(ignored by default)") : '';
    if (_local_file($medium)) {
	#- insert before first remote medium
	@{$urpm->{media}} = map {  
	    if (!_local_file($_) && !$inserted) {
		$inserted = 1;
		$urpm->{$options{on_the_fly} ? 'log' : 'info'}(N("adding medium \"%s\" before remote medium \"%s\"", $name, $_->{name}) . $ignore_text);
		$medium, $_;
	    } else { $_ }
	} @{$urpm->{media}};
    }
    if (!$inserted) {
	$urpm->{$options{on_the_fly} ? 'log' : 'info'}(N("adding medium \"%s\"", $name) . $ignore_text);
	push @{$urpm->{media}}, $medium;
    }

    $urpm->{modified} = 1;

    $name;
}

sub _register_media_cfg {
    my ($urpm, $url, $mirrorlist, $distribconf, $media_cfg) = @_;

    my $arch = $distribconf->getvalue('media_info', 'arch') || '';
    my $branch = $distribconf->getvalue('media_info', 'branch') || '';
    my $product = $distribconf->getvalue('media_info', 'product') || '';
    my $version = $distribconf->getvalue('media_info', 'version') || '';
    #official mirrors define $branch but not $product, other RPM repos do the
    #opposite :-/
    my $media_dir = (($branch or $product) . '-' . $version . '-' . $arch);
    $media_dir =~ tr!/!-!;
    my $media_path = $urpm->{mediacfgdir} . '/' . $media_dir;
    require File::Path;
    File::Path::mkpath($media_path);
    copy_and_own($media_cfg, $media_path . '/media.cfg')
	or $urpm->{info}(1, N("failed to copy media.cfg to %s (%d)", $media_path, $? >> 8)); 
    if ($url) {
	my $filename = $media_path . "/url";
	my @urls = split(/\n/, scalar cat_($filename));
	if (!grep { $url eq $_ } @urls) { 
	    append_to_file($filename, $url . "\n");
	}
    }
    if ($mirrorlist) {
	if ($mirrorlist ne '$MIRRORLIST') {
	    require urpm::cfg;
	    $mirrorlist = urpm::cfg::expand_line($mirrorlist);
	}
	my $filename = $media_path . "/mirrorlist";
	my @mirrorlists = split(/\n/, scalar cat_($filename));
	if (!grep { $mirrorlist eq $_ } @mirrorlists) { 
	    append_to_file($filename, $mirrorlist . "\n");
	}
    }
}

#- add distribution media, according to url given.
#- returns the list of names of added media.
#- options :
#- - initial_number : when adding several numbered media, start with this number
#- - probe_with : force use of rpms instead of using synthesis
#- - ask_media : callback to know whether each media should be added
#- - only_updates : only add "update" media (used by rpmdrake)
#- - mirrorlist
#- other options are passed to add_medium(): ignore, nolock, virtual
sub add_distrib_media {
    my ($urpm, $name, $url, %options) = @_;

    #- make sure configuration has been read.
    $urpm->{media} or die "caller should have used ->read_config or ->configure first";

    my $distribconf;

    if ($url && urpm::is_local_url($url)) {
	$url = _migrate_removable_url($url) or return();
	my $m = { url => $url };
	urpm::removable::try_mounting_medium_($urpm, $m) or $urpm->{error}(N("directory %s does not exist", $url));

	$distribconf = MDV::Distribconf->new(file_from_file_url($url) || $url, undef);
	$distribconf->settree('mandriva');

	my $dir = file_from_local_medium($m);
	my $media_cfg = reduce_pathname("$dir/" . $distribconf->getpath(undef, 'infodir') . '/media.cfg');
	$distribconf->parse_mediacfg($media_cfg)
	    or $urpm->{error}(N("this location doesn't seem to contain any distribution")), return ();
        if (!$options{virtual}) {
            _register_media_cfg($urpm, $dir, undef, $distribconf, $media_cfg);
        }
    } else {
	if ($options{mirrorlist}) {
	    $url and die "unexpected url $url together with mirrorlist $options{mirrorlist}\n";
	}

	my $m = { mirrorlist => $options{mirrorlist}, url => $url };
	my $parse_ok;
	try__maybe_mirrorlist($urpm, $m, 'probe', sub {
	    my $media_cfg = "$urpm->{cachedir}/partial/media.cfg";
	    $distribconf = _new_distribconf_and_download($urpm, $m);
	    $parse_ok = $distribconf && $distribconf->parse_mediacfg($media_cfg);
            if ($parse_ok && !$options{virtual}) {
	        _register_media_cfg($urpm, urpm::cfg::expand_line($m->{url}), $options{mirrorlist}, $distribconf, $media_cfg);
            }
	    $parse_ok;
	});
	$url = $m->{url};

	if ($distribconf) {
	    $parse_ok or $urpm->{error}(N("unable to parse media.cfg")), return();
	} else {   
	    $urpm->{error}(N("...retrieving failed: %s", $@));
	    $urpm->{error}(N("unable to access the distribution medium (no media.cfg file found)"));
	    return ();
	}
    }

    #- cosmetic update of name if it contains spaces.
    $name =~ /\s/ and $name .= ' ';

    my @newnames;
    #- at this point, we have found a media.cfg file, so parse it
    #- and create all necessary media according to it.
    my $medium_index = $options{initial_number} || 1;

    require urpm::mirrors;
    my $product_id = urpm::mirrors::parse_LDAP_namespace_structure(cat_('/etc/product.id'));

    foreach my $media ($distribconf->listmedia) {
        my $media_name = $distribconf->getvalue($media, 'name') || '';

        my @media_types = split(':', $distribconf->getvalue($media, 'media_type'));
        if ($product_id->{product} eq 'Free') {
            if (member('non-free', @media_types)) {
                $urpm->{log}(N("skipping non-free medium `%s'", $media));
                next;
            }
        }

        if (my $media_arch = $distribconf->getvalue($media, 'arch')) {
            if (!URPM::archscore($media_arch)) {
                $urpm->{log}(N("skipping non compatible media `%s' (for %s)",
			       $media, $media_arch));
                next;
            }
        }

	my $is_update_media = $distribconf->getvalue($media, 'updates_for');
	if ($options{only_updates}) {
	    $is_update_media or next;
	}

        my $add_by_default = !$distribconf->getvalue($media, 'noauto');
	my $ignore;
        if ($options{ask_media}) {
            $options{ask_media}->($media_name, $add_by_default) or next;
        } else {
	    my $simple_rpms = !$distribconf->getvalue($media, 'rpms');
	    $add_by_default || $simple_rpms or next;
	    $ignore = !$add_by_default;
	}

	my $use_copied_synthesis = urpm::is_cdrom_url($url) || $urpm->{options}{use_copied_hdlist} || $distribconf->getvalue($media, 'use_copied_hdlist');
	my $with_synthesis = $use_copied_synthesis && offset_pathname(
		$url,
		$distribconf->getpath($media, 'path'),
	    ) . '/' . $distribconf->getpath($media, 'synthesis');

	push @newnames, add_medium($urpm,
	    $name ? "$media_name ($name$medium_index)" : $media_name,
	    reduce_pathname($distribconf->getfullpath($media, 'path')),
	    $with_synthesis, 
	    !$use_copied_synthesis ? (media_info_dir => 'media_info') : @{[]},
	    !$use_copied_synthesis && $options{probe_with} ? ($options{probe_with} => 1) : (),
	    index_name => $name ? undef : 0,
	    $ignore ? (ignore => 1) : @{[]},
	    %options,
	    # the following override %options
	    $options{mirrorlist} ? ('with-dir' => $distribconf->getpath($media, 'path')) : (),
	    update => $is_update_media ? 1 : undef,
	);
	++$medium_index;
    }
    return @newnames;
}

sub _new_distribconf_and_download {
    my ($urpm, $medium) = @_;

    my $distribconf = MDV::Distribconf->new($medium->{url}, undef);
    $distribconf->settree('mandriva');

    $urpm->{log}(N("retrieving media.cfg file..."));
    my $url = $medium->{url};
    $medium->{url} = urpm::cfg::expand_line($url);
    urpm::download::sync_rel_one($urpm, $medium, $distribconf->getpath(undef, 'infodir') . '/media.cfg',
				 quiet => 1, preclean => 1) or return;
    $medium->{url} = urpm::cfg::substitute_back($medium->{url}, $url);
    $distribconf;
}

#- deprecated, use select_media_by_name instead
sub select_media {
    my $urpm = shift;
    my $options = {};
    if (ref $_[0]) { $options = shift }
    foreach (select_media_by_name($urpm, [ @_ ], $options->{strict_match})) {
	#- select medium by setting the modified flag, do not check ignore.
	$_->{modified} = 1;
    }
}

sub select_media_by_name {
    my ($urpm, $names, $b_strict_match) = @_;

    my %wanted = map { $_ => 1 } @$names;

    #- first the exact matches
    my @l = grep { delete $wanted{$_->{name}} } @{$urpm->{media}};

    #- check if some arguments don't correspond to the medium name.
    #- in such case, try to find the unique medium (or list candidate
    #- media found).
    foreach (keys %wanted) {
	my $q = quotemeta;
	my (@found, @foundi);
	my $regex  = $b_strict_match ? qr/^$q$/  : qr/$q/;
	my $regexi = $b_strict_match ? qr/^$q$/i : qr/$q/i;
	foreach my $medium (@{$urpm->{media}}) {
	    $medium->{name} =~ $regex  and push @found, $medium;
	    $medium->{name} =~ $regexi and push @foundi, $medium;
	}
	@found = @foundi if !@found;

	if (@found == 0) {
	    $urpm->{error}(N("trying to select nonexistent medium \"%s\"", $_));
	} else {
	    if (@found > 1) {
		$urpm->{log}(N("selecting multiple media: %s", join(", ", map { qq("$_->{name}") } @found)));
	    }
	    #- changed behaviour to select all occurences by default.
	    push @l, @found;
	}
    }
    @l;
}

#- deprecated, use remove_media instead
sub remove_selected_media {
    my ($urpm) = @_;

    remove_media($urpm, [ grep { $_->{modified} } @{$urpm->{media}} ]);
}

sub _remove_medium_from_mediacfg {
    my ($urpm, $mediacfg_dir, $url, $is_mirrorlist) = @_;

    my $filename = $mediacfg_dir;
    $filename .= $is_mirrorlist ? "/mirrorlist" : "/url";

    my @urls = split(/\n/, scalar cat_($filename));
    $urpm->{debug} and $urpm->{debug}("removing $url from $filename");
    output_safe($filename, join('\n', grep { $url ne $_ } @urls));
}

sub _cleanup_mediacfg_dir {
    my ($urpm, $to_remove) = @_;

    foreach my $medium (@$to_remove) {
	$medium->{mediacfg} or next;
	#this should never happen but dirname(undef) returns . on which we call
	#clean_dir so better be safe than sorry
	$medium->{mediacfg}[0]{root} or next;
	my $dir = reduce_pathname(dirname($medium->{mediacfg}[0]{root}));
	begins_with($medium->{mediacfg}[0]{root}, $dir) or next;
	if (!grep { $_->{mediacfg}[0]{root} == $medium->{mediacfg}[0]{root} } @{$urpm->{media}}) {
	    $urpm->{debug} and $urpm->{debug}("removing no longer used $dir");
	    -d $dir and urpm::sys::clean_dir($dir);
	    next;
	}

	if ($medium->{mirrorlist}) {
	    if (!grep { $_->{mirrorlist} eq $medium->{mirrorlist} } @{$urpm->{media}}) {
		_remove_medium_from_mediacfg($urpm, $dir, $medium->{mirrorlist}, 1);
	    }
	} elsif ($medium->{url}) {
	    if (!grep { $_->{url} eq $medium->{url} } @{$urpm->{media}}) {
		_remove_medium_from_mediacfg($urpm, $dir, $medium->{url}, 0);
	    }
	}
    }
}

sub remove_media {
    my ($urpm, $to_remove) = @_;

    foreach my $medium (@$to_remove) {
	$urpm->{info}(N("removing medium \"%s\"", $medium->{name}));

	#- mark to re-write configuration.
	$urpm->{modified} = 1;

	_clean_statedir_medium_files($urpm, $medium);

	#- remove proxy settings for this media
	urpm::download::remove_proxy_media($medium->{name});
    }
    $urpm->{media} = [ difference2($urpm->{media}, $to_remove) ];
    _cleanup_mediacfg_dir($urpm, $to_remove);
}

sub _clean_statedir_medium_files {
    my ($urpm, $medium) = @_;

    #- remove files associated with this medium.
    unlink grep { $_ } map { old_statedir_media_info_file($urpm, $medium, $_->[0], $_->[1]) } @media_info_prefix_suffix;

    my $dir = statedir_media_info_dir($urpm, $medium);
    -d $dir and urpm::sys::clean_dir($dir);

    remove_user_media_info_files($urpm, $medium);
}

sub _probe_with_try_list {
    my ($urpm, $medium, $f) = @_;

    $medium->{mirrorlist} and die "_probe_with_try_list does not handle mirrorlist\n";

    my @media_info_dirs = ('media_info', '.');

    my $base = file_from_local_medium($medium) || $medium->{url};

    foreach my $media_info_dir (@media_info_dirs) {
	my $file = "$media_info_dir/synthesis.hdlist.cz";
	my $url = reduce_pathname("$base/$file");
	if ($f->($url, $file)) {
	    $urpm->{debug} and $urpm->{debug}("found synthesis: $url");
	    $medium->{media_info_dir} = $media_info_dir;
	    delete $medium->{unknown_media_info};
	    return 1;
	}
    }
    undef;
}

sub may_reconfig_urpmi {
    my ($urpm, $medium) = @_;

    $medium->{url} && !urpm::is_cdrom_url($medium->{url}) or return; # we should handle mirrorlist?

    my $f;
    if (my $dir = file_from_file_url($medium->{url})) {
	$f = reduce_pathname("$dir/reconfig.urpmi");
    } else {
	$f = urpm::download::sync_rel_one($urpm, $medium, 'reconfig.urpmi', 
					  quiet => 1, preclean => 1) or return;
    }
    my $reconfigured = -s $f && reconfig_urpmi($urpm, $f, $medium);
    unlink $f if !is_local_medium($medium);
    $reconfigured;
}

#- read a reconfiguration file for urpmi, and reconfigure media accordingly
#- $rfile is the reconfiguration file (local), $name is the media name
#-
#- the format is similar to the RewriteRule of mod_rewrite, so:
#-    PATTERN REPLACEMENT [FLAG]
#- where FLAG can be L or N
#-
#- example of reconfig.urpmi:
#-    # this is an urpmi reconfiguration file
#-    /cooker /cooker/$ARCH
sub reconfig_urpmi {
    my ($urpm, $rfile, $medium) = @_;
    -r $rfile or return;

    my ($magic, @lines) = cat_($rfile);
    #- the first line of reconfig.urpmi must be magic, to be sure it's not an error file
    $magic =~ /^# this is an urpmi reconfiguration file/ or return undef;

    $urpm->{info}(N("reconfiguring urpmi for media \"%s\"", $medium->{name}));

    my @replacements;
    foreach (@lines) {
	chomp;
	s/^\s*//; s/#.*$//; s/\s*$//;
	$_ or next;
	my ($p, $r, $f) = split /\s+/, $_, 3;
	push @replacements, [ quotemeta $p, $r, $f || 1 ];
    }

    my $reconfigured = 0;
    my @reconfigurable = qw(url with_synthesis media_info_dir);

    my %orig = %$medium;

  URLS:
    foreach my $k (@reconfigurable) {
	foreach my $r (@replacements) {
	    if ($medium->{$k} =~ s/$r->[0]/$r->[1]/) {
		$reconfigured = 1;
		#- Flags stolen from mod_rewrite: L(ast), N(ext)
		if ($r->[2] =~ /L/) {
		    last;
		} elsif ($r->[2] =~ /N/) { #- dangerous option
		    redo URLS;
		}
	    }
	}
	#- check that the new url exists before committing changes (local mirrors)
	my $file = urpm::file_from_local_url($medium->{$k});
	if ($file && !-e $file) {
	    %$medium = %orig;
	    $reconfigured = 0;
	    $urpm->{log}(N("...reconfiguration failed"));
	    return;
	}
    }

    if ($reconfigured) {
	$urpm->{log}(N("reconfiguration done"));
	$urpm->{modified} = 1;
    }
    $reconfigured;
}

#- names.<media_name> is used by external progs (namely for bash-completion)
sub _generate_medium_names {
    my ($urpm, $medium) = @_;

    -e statedir_names($urpm, $medium) and return;

    my $fh = urpm::sys::open_safe($urpm, ">", statedir_names($urpm, $medium)) or return;

    foreach ($medium->{start} .. $medium->{end}) {
	my $pkg = $urpm->{depslist}[$_] or 
	  $urpm->{error}(N("Error generating names file: dependency %d not found", $_)), return;

	print $fh $pkg->name . "\n";
    }
}

sub _guess_synthesis_suffix {
    my ($url) = @_;
    $url =~ m!\bmedia/(\w+)/*\Z! && $1;
}

sub _synthesis_suffix {
    my ($medium) = @_;
    $medium->{with_synthesis} =~ /synthesis\.hdlist(.*?)(?:\.src)?\.cz$/ ? $1 : '';
}

sub _medium_is_up_to_date {
    my ($urpm, $medium) = @_;

    unlink cachedir_with_synthesis($urpm, $medium);

    $urpm->{info}(N("medium \"%s\" is up-to-date", $medium->{name}));

    #- the medium is now considered not modified.
    $medium->{modified} = 0;
}

sub _parse_synthesis {
    my ($urpm, $medium, $synthesis_file, $o_callback) = @_;

    -e $synthesis_file or return;

    $urpm->{log}(N("examining synthesis file [%s]", $synthesis_file));
    ($medium->{start}, $medium->{end}) = 
      $urpm->parse_synthesis($synthesis_file, $o_callback ? (callback => $o_callback) : @{[]});
}

sub _parse_synthesis_or_ignore {
    my ($urpm, $medium, $o_callback) = @_;

    _parse_synthesis($urpm, $medium, any_synthesis($urpm, $medium), $o_callback) or
      _ignore_medium_on_parse_error($urpm, $medium);
}

sub is_valid_medium {
    my ($medium) = @_;
    defined $medium->{start} && defined $medium->{end};
}

sub _ignore_medium_on_parse_error {
    my ($urpm, $medium) = @_;

    $urpm->{error}(N("problem reading synthesis file of medium \"%s\"", $medium->{name}));
    $medium->{ignore} = 1;
}

sub _copy_media_info_file {
    my ($urpm, $medium, $prefix, $suffix) = @_;

    my $name = "$prefix$suffix";
    my $path = _synthesis_dir($medium) . "/$prefix" . _synthesis_suffix($medium) . $suffix;
    -e $path or $path = file_from_local_medium($medium) . "/media_info/$name";

    my $result_file = "$urpm->{cachedir}/partial/$name";
    if (-e $path) {
	$urpm->{log}(N("copying [%s] for medium \"%s\"...", $path, $medium->{name}));
	copy_and_own($path, $result_file)
	  or $urpm->{error}(N("...copying failed")), return;
    }
    -s $result_file && $result_file;
}

sub _get_pubkey__local {
    my ($urpm, $medium) = @_;

    _copy_media_info_file($urpm, $medium, 'pubkey', '');
}

sub _download_pubkey {
    my ($urpm, $medium) = @_;

    _download_media_info_file($urpm, $medium, 'pubkey', '', { quiet => 1 });
}

# known options: quiet, callback
sub _download_media_info_file {
    my ($urpm, $medium, $prefix, $suffix, $options) = @_;

    my $versioned_prefix = do {
	my $version = urpm::md5sum::versioned_media_info_file($urpm, $medium, "$prefix$suffix");
	$version and $options->{is_versioned} = 1;
	$version ? "$version-$prefix" : $prefix;
    };

    my $tmp = _download_media_info_file_raw($urpm, $medium, 
					    $versioned_prefix, $suffix, $options) or return;
    my $result = dirname($tmp) . "/$prefix$suffix";
    $tmp eq $result or rename($tmp, $result) or return;
    $result;
}

sub _download_media_info_file_raw {
    my ($urpm, $medium, $prefix, $suffix, $options) = @_;

    my $name = "$prefix$suffix";
    my $result_file = "$urpm->{cachedir}/partial/$name";
    my $found;
    if (_synthesis_suffix($medium)) {
	my $local_name = $prefix . _synthesis_suffix($medium) . $suffix;

	if (urpm::download::sync_rel_to($urpm, $medium, 
					_synthesis_dir_rel($medium) . "/$local_name", $result_file,
					%$options)) {
	    $found = 1;
	}
    }
    if (!$found) {
	urpm::download::sync_rel_one($urpm, $medium, _synthesis_dir_rel($medium) . "/$name", 
				     %$options);
    }
    -s $result_file && $result_file;
}

sub get_descriptions_local {
    my ($urpm, $medium) = @_;

    unlink statedir_descriptions($urpm, $medium);

    my $dir = file_from_local_medium($medium);
    my $description_file = "$dir/media_info/descriptions"; #- new default location
    -e $description_file or $description_file = "$dir/../descriptions";
    -e $description_file or return;

    $urpm->{log}(N("copying description file of \"%s\"...", $medium->{name}));
    if (copy_and_own($description_file, statedir_descriptions($urpm, $medium))) {
	$urpm->{log}(N("...copying done"));
    } else {
	$urpm->{error}(N("...copying failed"));
	$medium->{ignore} = 1;
    }
}
#- not handling different mirrors since the file is not always available
sub get_descriptions_remote {
    my ($urpm, $medium) = @_;

    if (-e statedir_descriptions($urpm, $medium)) {
	unlink "$urpm->{cachedir}/partial/descriptions";
	urpm::sys::move_or_die($urpm, statedir_descriptions($urpm, $medium), "$urpm->{cachedir}/partial/descriptions");
    }
    my $result = urpm::download::sync_rel_one($urpm, $medium, 'media_info/descriptions', quiet => 1, preclean => 1);

    if ($result) {
	urpm::sys::move_or_die($urpm, $result, statedir_descriptions($urpm, $medium));
    }
}
sub get_synthesis__local {
    my ($urpm, $medium, $callback) = @_;

    my $f = cachedir_with_synthesis($urpm, $medium);
    unlink $f;
    $urpm->{log}(N("copying [%s] for medium \"%s\"...", _url_with_synthesis($medium), $medium->{name}));
    $callback and $callback->('copy', $medium->{name});
    if (copy_and_own(_url_with_synthesis($medium), $f)) {
	$callback and $callback->('done', $medium->{name});
	$urpm->{log}(N("...copying done"));
	if (file_size($f) < 20) {
	    $urpm->{error}(N("copy of [%s] failed (file is suspiciously small)", $f));
	    0;
	} else {
	    1;
	}
    } else {
	$callback and $callback->('failed', $medium->{name});
	#- force error, reported afterwards
	unlink $f;
	0;
    }
}
sub get_synthesis__remote {
    my ($urpm, $medium, $is_a_probe, $options) = @_;

    my $ok = try__maybe_mirrorlist($urpm, $medium, $is_a_probe, sub {
	_download_media_info_file($urpm, $medium, 'synthesis.hdlist', '.cz',
				  $options)
	    && _check_synthesis(cachedir_with_synthesis($urpm, $medium));
    });
    if (!$ok) {
	chomp(my $err = $@);
	$urpm->{error}(N("...retrieving failed: %s", $err));
    }

    $ok &&= check_synthesis_md5sum($urpm, $medium) if !$options->{force} && !$options->{nomd5sum};

    $ok;
}

sub _check_synthesis {
    my ($synthesis_file) = @_;

    file_size($synthesis_file) >= 20 or return;

    # check first 2 lines do not contain typical html code
    # this is useful for servers not returning a valid HTTP error (#39918)
    open(my $F, '<', $synthesis_file) or return;
    my $s = <$F>; $s .= <$F>; 
    $s !~ /<html>|<!DOCTYPE\s/i;
}

#- check copied/downloaded file has right signature.
sub check_synthesis_md5sum {
    my ($urpm, $medium) = @_;

    my $wanted_md5sum = urpm::md5sum::from_MD5SUM__or_warn($urpm, $medium->{parsed_md5sum}, 'synthesis.hdlist.cz');
    if ($wanted_md5sum) {
	$urpm->{log}(N("computing md5sum of retrieved source synthesis"));
	urpm::md5sum::compute(cachedir_with_synthesis($urpm, $medium)) eq $wanted_md5sum or
	    $urpm->{error}(N("retrieval of [%s] failed (md5sum mismatch)", _url_with_synthesis($medium))), return;
    }
    1;
}

sub _call_genhdlist2 {
    my ($urpm, $medium) = @_;

    !$medium->{with_synthesis} or $urpm->{fatal}(1, 'with_synthesis not handled with --probe-rpms');

    my $dir = file_from_local_medium($medium);

    system('genhdlist2', 
	   $urpm->{debug} ? '--verbose' : @{[]}, '--no-hdlist',
	   '--media_info-dir', "$urpm->{cachedir}/partial", $dir) == 0 
      or $urpm->{error}(N("genhdlist2 failed on %s", $dir)), return;

    1;
}

sub _is_statedir_MD5SUM_uptodate {
    my ($urpm, $medium, $new_MD5SUM) = @_;

    my $current_MD5SUM = statedir_MD5SUM($urpm, $medium);

    $urpm->{log}(N("comparing %s and %s", $new_MD5SUM, $current_MD5SUM));

    cat_($new_MD5SUM) eq cat_($current_MD5SUM);
}

#- options: callback, force, nomd5sum, probe_with
sub _update_medium__parse_if_unmodified__local {
    my ($urpm, $medium, $options) = @_;

    if ($options->{probe_with} ne 'rpms') {
	#- the directory given does not exist and may be accessible
	#- by mounting some other directory. Try to figure it out and mount
	#- everything that might be necessary.
	urpm::removable::try_mounting_medium($urpm, $medium) or return;
    }

    #- check for a reconfig.urpmi file (if not already reconfigured)
    if (!$medium->{noreconfigure}) {
	may_reconfig_urpmi($urpm, $medium);
    }

    #- try to probe for possible with_synthesis parameter, unless
    #- it is already defined (and valid).
    if (!_valid_synthesis_dir($medium) && $options->{probe_with} ne 'rpms') {
	_probe_with_try_list($urpm, $medium, sub {
	    my ($url) = @_;
	    -e $url or return;
	    if (file_size($url) >= 20) {
		1;
	    } else {
		$urpm->{error}(N("invalid hdlist file %s for medium \"%s\"", $url, $medium->{name}));
		0;
	    }
	});
    }

    if (_is_local_virtual($medium)) {
	#- syncing a local virtual medium is very simple :)
	1;
    } elsif ($options->{probe_with} eq 'rpms' || !_valid_synthesis_dir($medium)) {
	_call_genhdlist2($urpm, $medium) or return '';
	if (!$medium->{'no-media-info'}) {
	    $medium->{'no-media-info'} = 1;
	    $urpm->{modified} = 1;
	}
	1;
    } elsif (_valid_synthesis_dir($medium)) {
	my $new_MD5SUM = _synthesis_dir($medium) . '/MD5SUM';
	unlink "$urpm->{cachedir}/partial/MD5SUM";

	if (!$options->{nomd5sum} && file_size($new_MD5SUM) > 32) {	
	    if (!$options->{force} && _is_statedir_MD5SUM_uptodate($urpm, $medium, $new_MD5SUM)) {
	        _medium_is_up_to_date($urpm, $medium);
		return 'unmodified';
	    }

	    $urpm->{log}(N("copying MD5SUM file of \"%s\"...", $medium->{name}));
	    copy_and_own($new_MD5SUM, "$urpm->{cachedir}/partial/MD5SUM");
	    $medium->{parsed_md5sum} = urpm::md5sum::parse($new_MD5SUM);
	}

	my $ok = get_synthesis__local($urpm, $medium, $options->{callback});
	$ok &&= check_synthesis_md5sum($urpm, $medium) if !$options->{force} && !$options->{nomd5sum};
	
	if ($ok) {
	    1;
	} elsif ($urpm->{options}{'build-hdlist-on-error'}) {
	    #- if copying synthesis has failed, try to build it directly.
	    _call_genhdlist2($urpm, $medium) or return '';
	    1;
	} else {
	    _ignore_medium_on_parse_error($urpm, $medium);
	    '';
	}
    }
}

sub _download_MD5SUM {
    my ($urpm, $medium) = @_;

    urpm::download::sync_rel_one($urpm, $medium, 
				 _synthesis_dir_rel($medium) . '/MD5SUM',
				 quiet => 1, preclean => 1);
}

sub _download_MD5SUM_and_check {
    my ($urpm, $medium, $is_a_probe) = @_;

    my ($err, $cachedir_MD5SUM);
    require urpm::mirrors;
    try__maybe_mirrorlist($urpm, $medium, $is_a_probe, sub {
	$cachedir_MD5SUM = _download_MD5SUM($urpm, $medium) or $err = $@;
	$cachedir_MD5SUM && urpm::md5sum::check_file($cachedir_MD5SUM);
    }) and return $cachedir_MD5SUM;

    if ($cachedir_MD5SUM) {
	$urpm->{error}(N("invalid MD5SUM file (downloaded from %s)", _synthesis_dir($medium)));
    } else {
	$urpm->{error}(N("...retrieving failed: %s", $err));
	$is_a_probe and $urpm->{error}(N("no metadata found for medium \"%s\"", $medium->{name}));
    }
    undef;
}

#- options: callback, ask_retry, force, nomd5sum, probe_with, quiet
sub _update_medium__parse_if_unmodified__remote {
    my ($urpm, $medium, $options) = @_;

    my $updating = -e statedir_synthesis($urpm, $medium);

    #- examine if a distant MD5SUM file is available.
	if (!$options->{nomd5sum}) {
	    my $new_MD5SUM = _download_MD5SUM_and_check($urpm, $medium, !$updating);

	    if (!$new_MD5SUM) {
		#- check for a reconfig.urpmi file (if not already reconfigured)
		if (!$medium->{noreconfigure}) {
		    may_reconfig_urpmi($urpm, $medium)
		      and goto &_update_medium__parse_if_unmodified__remote;
		}
		return;
	    }
	    if ($options->{force} < 2 && _is_statedir_MD5SUM_uptodate($urpm, $medium, $new_MD5SUM)) {
		_medium_is_up_to_date($urpm, $medium);
		return 'unmodified';
	    }
	    $medium->{parsed_md5sum} = urpm::md5sum::parse($new_MD5SUM);
	}

    #- try to probe for possible with_synthesis parameter, unless
    #- it is already defined (and valid).
    $urpm->{log}(N("retrieving source synthesis of \"%s\"...", $medium->{name}));
    $options->{callback} and $options->{callback}('retrieve', $medium->{name});
    my $error = sub {
	my ($msg) = @_;
	$urpm->{error}($msg);
	unlink cachedir_with_synthesis($urpm, $medium);
	$options->{callback} and $options->{callback}('failed', $medium->{name});
    };

	if ($options->{force}) {
	    unlink cachedir_with_synthesis($urpm, $medium);
	} else {
	    #- for rsync, try to sync (copy if needed) local copy after restored the previous one.
	    my $previous_synthesis = statedir_synthesis($urpm, $medium);
	    if (-e $previous_synthesis && urpm::protocol_from_url($medium->{url}) eq 'rsync') {
		copy_and_own(
		    $previous_synthesis,
		    cachedir_with_synthesis($urpm, $medium),
		) or $error->(N("...copying failed")), return;
	    }
	}
	my $ok = get_synthesis__remote($urpm, $medium, !$updating, $options);

    $options->{callback} and $options->{callback}('done', $medium->{name});

    if (!$ok) {
	_ignore_medium_on_parse_error($urpm, $medium);
	return;
    }
    1;
}

sub _get_pubkey {
    my ($urpm, $medium, $b_wait_lock) = @_;
  
    my $local = file_from_local_medium($medium);

    #- examine if a pubkey file is available.
    ($local ? \&_get_pubkey__local : \&_download_pubkey)->($urpm, $medium);

    $medium->{'key-ids'} = _read_cachedir_pubkey($urpm, $medium, $b_wait_lock);
    $urpm->{modified} = 1;
}

sub _get_descriptions {
    my ($urpm, $medium) = @_;

    my $local = file_from_local_medium($medium);

    # do not get "descriptions" on non "update" media since it's useless and potentially slow
    if ($medium->{update}) {
	($local ? \&get_descriptions_local : \&get_descriptions_remote)->($urpm, $medium);
    }
}

# options: wait_lock, nopubkey, forcekey
sub _may_get_pubkey {
    my ($urpm, $medium, %options) = @_;
    
    _get_pubkey($urpm, $medium, $options{wait_lock}) if !$options{nopubkey} && (!$medium->{'key-ids'} || $options{forcekey});
}

sub _read_cachedir_pubkey {
    my ($urpm, $medium, $b_wait_lock) = @_;
    -s "$urpm->{cachedir}/partial/pubkey" or return;

    $urpm->{log}(N("examining pubkey file of \"%s\"...", $medium->{name}));

    my $_rpm_lock = urpm::lock::rpm_db($urpm, 'exclusive', wait => $b_wait_lock);
    my $db = urpm::db_open_or_die_($urpm, 'rw');

    my %key_ids;
    URPM::import_needed_pubkeys_from_file($db,
	"$urpm->{cachedir}/partial/pubkey",
	sub {
	    my ($id, $imported) = @_;
	    if ($id) {
		$key_ids{$id} = undef;
		$imported and $urpm->{log}(N("...imported key %s from pubkey file of \"%s\"",
					     $id, $medium->{name}));
		$imported or $urpm->{debug}("pubkey $id already imported") if $urpm->{debug};
	    } else {
		$urpm->{error}(N("unable to import pubkey file of \"%s\"", $medium->{name}));
	    }
	});

    unlink "$urpm->{cachedir}/partial/pubkey";
    
    join(',', keys %key_ids);
}

#- options: callback, ask_retry, force, nomd5sum, probe_with, quiet, forcekey, nopubkey, wait_lock
#- (from _update_medium__parse_if_unmodified__local and _update_medium__parse_if_unmodified__remote)
sub _update_medium_ {
    my ($urpm, $medium, %options) = @_;

    #- always delete a remaining list file or pubkey file in cache.
    foreach (qw(list pubkey)) {
	unlink "$urpm->{cachedir}/partial/$_";
    }

    _pick_mirror_if_needed($urpm, $medium, 'allow-cache-update');

    {
	my $rc = 
	  is_local_medium($medium)
	    ? _update_medium__parse_if_unmodified__local($urpm, $medium, \%options)
	    : _update_medium__parse_if_unmodified__remote($urpm, $medium, \%options);

	if ($options{forcekey} && $rc eq 'unmodified') {
	    _get_pubkey($urpm, $medium, $options{wait_lock}); # we must do it now, quite hackish...
	    return 1;
	}

	if (!$rc || $rc eq 'unmodified') {
	    return $rc;
	}
    }

    my $is_updating = -e statedir_synthesis($urpm, $medium);

    if (!_is_local_virtual($medium)) {
	if (file_size(cachedir_with_synthesis($urpm, $medium)) < 20) {
	    $urpm->{error}(N("no synthesis file found for medium \"%s\"", $medium->{name}));
	    return;
	}

	    #- use new files

	    unlink statedir_synthesis($urpm, $medium);
	    urpm::sys::move_or_die($urpm, cachedir_with_synthesis($urpm, $medium),
				    statedir_synthesis($urpm, $medium));

	    unlink statedir_MD5SUM($urpm, $medium);
	    if (!$medium->{with_synthesis}) { # no MD5SUM when using with_synthesis, urpmi.update will update everytime!
		urpm::sys::move_or_die($urpm, "$urpm->{cachedir}/partial/MD5SUM",
				 statedir_MD5SUM($urpm, $medium)) if -e "$urpm->{cachedir}/partial/MD5SUM";
	    }

	# we never download hdlist by default. urpmf will download it via any_hdlist() if really needed
	unlink statedir_hdlist($urpm, $medium);

	remove_user_media_info_files($urpm, $medium);

	if (!_local_file($medium)) {
	    _retrieve_xml_media_info_or_remove($urpm, $medium, $options{quiet}) or return;
	}
    }
    $medium->{modified} = 0;

    # generated on first _parse_media()
    unlink statedir_names($urpm, $medium);

    _get_descriptions($urpm, $medium);
    _may_get_pubkey($urpm, $medium, %options);

    $is_updating and $urpm->{info}(N("updated medium \"%s\"", $medium->{name}));

    1;
}

sub _update_medium {
    my ($urpm, $medium, %options) = @_;

    my $rc = _update_medium_($urpm, $medium, %options);

    if (!$rc && !_is_local_virtual($medium)) {
	#- an error has occured for updating the medium, we have to remove temporary files.
	unlink(glob("$urpm->{cachedir}/partial/*"));
    }
    $rc;
}

#- Update the urpmi database w.r.t. the current configuration.
#- Takes care of modifications, and tries some tricks to bypass
#- the recomputation of base files.
#- Recognized options :
#-   all         : all medias are being rebuilt
#- allow_failures: whereas failing to update a medium is non fatal
#-   ask_retry   : function called when a download fails. if it returns true, the download is retried
#-   callback    : UI callback
#-   forcekey    : force retrieval of pubkey
#-   force       : try to force rebuilding base files
#-   nomd5sum    : don't verify MD5SUM of retrieved files
#-   nopubkey    : don't use rpm pubkeys
#-   probe_with  : probe synthesis or rpms
#-   quiet       : download synthesis quietly
#-   wait_lock   : block until lock can be acquired
sub update_media {
    my ($urpm, %options) = @_;

    $urpm->{media} or return; # verify that configuration has been read

    if ($options{all}) {
	$_->{modified} ||= 1 foreach all_media_to_update($urpm);
    }

    update_those_media($urpm, [ grep { $_->{modified} } non_ignored_media($urpm) ], %options);
}

sub update_those_media {
    my ($urpm, $media, %options) = @_;

    $options{nopubkey} ||= $urpm->{options}{nopubkey};

    #- examine each medium to see if one of them needs to be updated.
    #- if this is the case and if not forced, try to use a pre-calculated
    #- synthesis file, else build it from rpm files.
    clean($urpm);

    my %updates_result;
    foreach my $medium (@$media) {

	#- don't ever update static media
	$medium->{static} and next;

	my $unsubstituted_url = $medium->{url};
	$medium->{url} = urpm::cfg::expand_line($medium->{url}) if $medium->{url};
	my $rc = _update_medium($urpm, $medium, %options);
	$medium->{url} = urpm::cfg::substitute_back($medium->{url}, $unsubstituted_url);
	$rc or return if !$options{allow_failures};
	$updates_result{$rc || 'error'}++;
    }

    $urpm->{debug} and $urpm->{debug}('update_medium: ' . join(' ', map { "$_=$updates_result{$_}" } keys %updates_result));

    if ($updates_result{1} == 0) {
	#- only errors/unmodified, leave now
	#- (this ensures buggy added medium is not added to urpmi.cfg)
	return $updates_result{error} == 0;
    }

    if ($urpm->{modified}) {
	#- write config files in any case
	write_config($urpm);
	urpm::download::dump_proxy_config();
    }

    $updates_result{error} == 0;
}

sub _maybe_in_statedir_MD5SUM {
    my ($urpm, $medium, $file) = @_;
   
    my $md5sum_file = statedir_MD5SUM($urpm, $medium);
    -e $md5sum_file && urpm::md5sum::parse($md5sum_file)->{$file};
}

sub _retrieve_xml_media_info_or_remove {
    my ($urpm, $medium, $quiet) = @_;

    my $ok = 1;

    foreach my $xml_info (@xml_media_info) {
	my $f = statedir_xml_info($urpm, $medium, $xml_info);

	my $get_it = urpm::is_cdrom_url($medium->{url}) ||
		 get_medium_option($urpm, $medium, 'xml-info') eq 'always' ||
		 get_medium_option($urpm, $medium, 'xml-info') eq 'update-only' && -e $f;
	if ($get_it && _maybe_in_statedir_MD5SUM($urpm, $medium, "$xml_info.xml.lzma")) {
	    $ok &&= _retrieve_media_info_file_and_check_MD5SUM($urpm, $medium, $xml_info, '.xml.lzma', $quiet);
	    $ok = 1 if urpm::is_cdrom_url($medium->{url});
	} else {
	    #- "on-demand"
	    unlink $f;
	}
    }
    $ok;
}

sub _retrieve_media_info_file_and_check_MD5SUM {
    my ($urpm, $medium, $prefix, $suffix, $quiet) = @_;

    my $name = "$prefix$suffix";
    my $cachedir_file = 
      is_local_medium($medium) ?
	_copy_media_info_file($urpm, $medium, $prefix, $suffix) :
	_download_media_info_file($urpm, $medium, $prefix, $suffix, { quiet => $quiet, callback => \&urpm::download::sync_logger }) or
	  $urpm->{error}(N("retrieval of [%s] failed", _synthesis_dir($medium) .  "/$name")), return;

    my $wanted_md5sum = urpm::md5sum::from_MD5SUM__or_warn($urpm, $medium->{parsed_md5sum}, $name);
    if ($wanted_md5sum) {
	$urpm->{debug}("computing md5sum of retrieved $name") if $urpm->{debug};
	urpm::md5sum::compute($cachedir_file) eq $wanted_md5sum or
	    $urpm->{error}(N("retrieval of [%s] failed (md5sum mismatch)", _synthesis_dir($medium) .  "/$name")), return;

	urpm::util::move($cachedir_file, statedir_media_info_file($urpm, $medium, $prefix, $suffix)) or return;
    }
    1;
}

sub _download_temp_md5sum_and_parse {
    my ($urpm, $medium) = @_;

    $urpm->{debug}("downloading MD5SUM to know updated versioned metadata filename") if $urpm->{debug};
    my $md5sum_file = _download_MD5SUM($urpm, $medium);
    urpm::md5sum::parse($md5sum_file);
}

sub _any_media_info__or_download {
    my ($urpm, $medium, $prefix, $suffix, $quiet, $o_callback) = @_;

    my $f = statedir_media_info_file($urpm, $medium, $prefix, $suffix);
    -s $f and return $f;

    if ($<) {
	urpm::ensure_valid_cachedir($urpm);
	$f = "$urpm->{cachedir}/" . statedir_media_info_basename($medium, $prefix, $suffix);
	-s $f and return $f;
    }

    get_medium_option($urpm, $medium, 'xml-info') ne 'never' or return;
   
    _maybe_in_statedir_MD5SUM($urpm, $medium, "$prefix$suffix") or return;

    $medium->{parsed_md5sum} ||= _download_temp_md5sum_and_parse($urpm, $medium);

    my $file_in_partial = 
      _download_media_info_file($urpm, $medium, $prefix, $suffix, 
				{ quiet => $quiet, callback => $o_callback }) or return;

    urpm::util::move($file_in_partial, $f) or return;

    $f;
}

#- side-effects:
#-   + those of urpm::mirrors::pick_one ($urpm->{mirrors_cache}, $medium->{url})
sub _pick_mirror_if_needed {
    my ($urpm, $medium, $allow_cache_update) = @_;

    $medium->{mirrorlist} && !$medium->{url} or return;

    require urpm::mirrors;
    urpm::mirrors::pick_one($urpm, $medium, $allow_cache_update);
}

#- side-effects:
#-   + those of urpm::mirrors::try ($urpm->{mirrors_cache}, $medium->{url})
sub try__maybe_mirrorlist {
    my ($urpm, $medium, $is_a_probe, $try) = @_;

    if ($medium->{mirrorlist}) {
	if (urpm::download::use_metalink($urpm, $medium)) {
	    #- help things...
	    _pick_mirror_if_needed($urpm, $medium, 'allow-cache-update');

	    $try->();
	} else {
	    require urpm::mirrors;
	    $is_a_probe
	      ? urpm::mirrors::try_probe($urpm, $medium, $try)
	      : urpm::mirrors::try($urpm, $medium, $try);
	}
    } else {
	$try->();
    }
}

#- clean params and depslist computation zone.
sub clean {
    my ($urpm) = @_;

    $urpm->{depslist} = [];
    $urpm->{provides} = {};

    foreach (@{$urpm->{media} || []}) {
	delete $_->{start};
	delete $_->{end};
    }
}

1;