package BCD::Media; use File::Copy::Recursive qw(dircopy pathrm); use strict; use URPM; use File::Copy; use File::stat; use Data::Dumper; use MDK::Common; use File::Glob ':glob'; use Parallel::ForkManager; use Data::Dumper; use BCD::Common qw(:DEFAULT $isoconf $wd $name $arch $version $based_on $repo $builddir $distrib_path $error_color $DISTRIB $THEME $tmp_rpmsrate $NB_FORK $dir_deps @list_of_removed); our @ISA = qw(Exporter); our @EXPORT = qw(main_media list_media use_gendistrib list_medias_available check_repo_hdlist rpmcheck_launch solve_doble_issue clean_urpmq_log get_srpms_list find_all_kernel, create_mediacfg parse_synthesis); our @EXPORT_OK = qw($rpmcheck_list_pkg $file_rpmcheck_log); my $verbose; my $LOG="MEDIA -"; my $color = "magenta"; # all mounted loop dir to umount my @to_umount; my @add_media; my $mediainfo_sourcedir; if (defined($isoconf->{mediainfo}{fullpath})) { $mediainfo_sourcedir = $isoconf->{mediainfo}{fullpath}; } else { $mediainfo_sourcedir = "$repo/$based_on/$arch/$isoconf->{mediainfo}{mediainfo_dir}"; } my $chroot_path = "/tmp/$isoconf->{nameid}-$version-$isoconf->{arch}"; our $rpmcheck_list_pkg = "/tmp/rpmcheck_list_pkg_to_add_$isoconf->{nameid}-$version-$isoconf->{arch}"; our $file_rpmcheck_log = "/tmp/rpmcheck_failure_$isoconf->{nameid}-$version-$isoconf->{arch}"; sub create_media { print_color("$LOG Create the media dir in $builddir", $color); foreach my $media (@{$isoconf->{media}{list}}) { create_needed_dir("$builddir/media/$media->{destmedia}"); } } sub show_size { print_color("$LOG size of the builddir", $color); system("du -sLh $builddir"); } sub list_media { print_color("$LOG all media are listed there:", $color); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; print " name: $media->{name} desc: $media->{desc} mediadir: $media->{mediadir} destmedia: $media->{destmedia} pubkey: $media->{pubkey}\n"; foreach my $files (@{$media->{input}}) { $files->{file} or next; if (-f $files->{file}) { print "list of wanted: $files->{file}\n"; } else { print_color("$files->{file} not present", $error_color); } } foreach my $files (@{$media->{exclude}}) { $files->{file} or next; if (-f $files->{file}) { print "list of exclusion: $files->{file}\n"; } else { print_color("$files->{file} not present", $error_color); } } } } sub copy_pubkey { my ($source_key, $destmedia) = @_; print_color("$LOG copy $source_key to the $builddir/media/$destmedia/media_info directory", $color); create_needed_dir("$builddir/media/$destmedia/media_info/"); if (! -f $source_key) { print_color("$LOG can't find the $source_key", $error_color); #exit; } system("cp -vf $source_key $builddir/media/$destmedia/media_info/"); } sub use_gendistrib { my $destmedia; print_color("$LOG remove hdlists and synthesys from $builddir/media/$destmedia/media_info and $builddir/media/media_info", $color); system("rm -vf $builddir/media/media_info/*.cz"); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{destmedia} or next; if ($destmedia eq $media->{destmedia}) { next } $destmedia = $media->{destmedia}; system("rm -vf $builddir/media/$destmedia/media_info/*.cz"); system("rm -vf $builddir/media/$destmedia/media_info/MD5SUM"); } undef $destmedia; print_color("$LOG gendistrib --mediacfg $builddir/$isoconf->{iso}{mediacfg} $builddir", $color); my $cmd = "gendistrib --mediacfg $builddir/$isoconf->{iso}{mediacfg} $builddir"; system($cmd) == 0 or die "$LOG system $cmd failed: $?\n"; if ($isoconf->{based_on} lt "2009.1" || $isoconf->{based_on} eq "mes5") { foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; if ($destmedia eq $media->{destmedia}) { next } $destmedia = $media->{destmedia}; print_color("$LOG < 2009.1 or mes5 detected", $color); print_color("$LOG move hdlists and synthesys from $builddir/media/$destmedia/media_info to $builddir/media/media_info/", $color); system("rm -vf $builddir/media/media_info/*_$media->{name}.cz"); system("rm -vf $builddir/media/media_info/*_main.cz"); system("rm -vf $builddir/media/media_info/pubkey_$media->{name}"); system("cp -avf $builddir/media/$destmedia/media_info/hdlist.cz $builddir/media/media_info/hdlist_$media->{name}.cz"); system("cp -avf $builddir/media/$destmedia/media_info/pubkey $builddir/media/media_info/pubkey_$media->{name}"); system("cp -avf $builddir/media/$destmedia/media_info/synthesis.hdlist.cz $builddir/media/media_info/synthesis.hdlist_$media->{name}.cz"); system("rm -vf $builddir/media/$destmedia/media_info/*.cz"); system("rm -vf $builddir/media/$destmedia/media_info/MD5SUM"); } } } sub copy_rpm { my ($source_dir, $dest_dir) = @_; system ("cp -a $source_dir/*.rpm $dest_dir"); } sub create_mediacfg { print_color("$LOG $builddir/$isoconf->{iso}{mediacfg}", $color); system("rm -vrf $builddir/$isoconf->{mediainfo}{mediainfo_dir}"); create_needed_dir("$builddir/$isoconf->{mediainfo}{mediainfo_dir}"); my $media_cfg = "$builddir/$isoconf->{iso}{mediacfg}"; open my $MEDIACFG, ">$media_cfg"; print $MEDIACFG "\n"; print $MEDIACFG "[media_info]\n"; print $MEDIACFG "version=$version\n"; if ($isoconf->{based_on} gt "2009.1" && $isoconf->{based_on} ne "mes5") { print_color("$LOG > 2009.1 or not a mes5 product", $color); print $MEDIACFG "mediacfg_version=2\n"; } print $MEDIACFG "branch=$isoconf->{iso}{branch}\n"; print $MEDIACFG "arch=$arch\n"; print $MEDIACFG "minor=$isoconf->{iso}{minor}\n"; print $MEDIACFG "subversion=$isoconf->{iso}{subversion}\n"; if ($isoconf->{mediainfo}{suppl}{todo} eq "yes") { print $MEDIACFG "suppl=1\n"; } if ($isoconf->{mediainfo}{askmedia}{todo} eq "yes") { print $MEDIACFG "askmedia=1\n"; } if ($isoconf->{mediainfo}{xmlinfo}{todo} eq "yes") { print $MEDIACFG "xml-info=1\n"; } my $destmedia; foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; if ($destmedia eq $media->{destmedia}) { next } $destmedia = $media->{destmedia}; if ($isoconf->{based_on} =~ "mes5" || $isoconf->{based_on} lt "2009.1") { print_color("$LOG < 2009.1 or mes5 detected", $color); print $MEDIACFG " [$media->{destmedia}] hdlist=hdlist_$media->{name}.cz pubkey=pubkey_$media->{name} name=$isoconf->{iso}{header}{volumesetid} desc=$isoconf->{iso}{header}{volumesetid} \n"; } else { print $MEDIACFG " [$media->{destmedia}] synthesis=$media->{destmedia}/media_info/synthesis.hdlist.cz hdlist=$media->{destmedia}/media_info/hdlist.cz pubkey=$media->{destmedia}/media_info/pubkey name=$media->{desc} \n"; } } close $MEDIACFG; } sub parse_synthesis { my ($path) = @_; my $urpm = new URPM; my %hash_rpms; foreach (<$path/*.cz>) { $urpm->parse_synthesis($_, keep_all_tags => 0); $urpm->traverse(sub { my ($p) = @_; $hash_rpms{ $p->name } = [ $p->version, $p->release, ]; } ); } return %hash_rpms; } # based on a similar functions in pkgs.pm sub get_list_from_CAT { #4 META_CLASS"desktop" mandriva-xfce-config-One my ($file, $cat, $poids) = @_; my (%rates, @need_to_copy); my (@l); local $_; foreach (cat_($file)) { s/#.*//; # comments my ($indent, $data) = /(\s*)(.*)/; next if !$data; # skip empty lines @l = grep { $_->[0] < length $indent } @l; my @m = @l ? @{$l[-1][1]} : (); my ($t, $flag, @l2); while ($data =~ /^(( [1-5] | (?: (?: !\s*)? [0-9A-Z_]+(?:".*?")?) (?: \s*\|\|\s* (?: !\s*)? [0-9A-Z_]+(?:".*?")?)* ) (?:\s+|$) )(.*)/x) { #@")) { ($t, $flag, $data) = ($1,$2,$3); while ($flag =~ s,^\s*(("[^"]*"|[^"\s]*)*)\s+,$1,) {} push @m, $flag; push @l2, [ length $indent, [ @m ] ]; $indent .= $t; } if ($data) { # has packages on same line my ($rates, $flags) = partition { /^\d$/ } @m; foreach my $name (split ' ', $data) { if (member($cat, @$flags)) { if ("@$rates" ge "$poids") { push @need_to_copy, $name; } else { print "exclude $name of $cat (@$rates < $poids)\n"; } next; } } push @l, @l2; } else { push @l, [ $l2[0][0], $l2[-1][1] ]; } } \%rates, \@need_to_copy; } sub mini_chroot { # it is usefull to use a mini chroot for add remove media, we have to keep the list of installed packages to copy them on ISO print_color("$LOG create a mini chroot to run urpmi", $color); # store all file needed to be installed on the system my $cmd = "LC_ALL=C sudo urpmi --quiet --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root $chroot_path"; # sudo urpmi --no-verify-rpm --auto --urpmi-root /tmp/mk --use-distrib /mnt/BIG/dis/2009.0/i586/ basesystem-minimal urpmi --test # system("LC_ALL=C sudo urpmi --test --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root /tmp/$isoconf->{nameid}-$version-$isoconf->{arch} --media 'Main,Main Updates' --excludemedia 'Main Testing,Main Backports' urpmi basesystem-minimal bash basesystem > $chroot_path/$isoconf->{tocopy_file}_TODO_chroot"); # install all those file to be able to use urpmi.XXXXXX system("LC_ALL=C $cmd --prefer $DISTRIB,$THEME urpmi bash"); system("sudo chroot $chroot_path mkdir -p $distrib_path"); system("sudo mount -o bind $repo/$based_on/$arch $chroot_path/$distrib_path -o ro"); push @to_umount, $distrib_path; system("sudo chroot $chroot_path urpmi.removemedia -a"); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; chomp($media->{name}); system("LC_ALL=C sudo chroot $chroot_path urpmi.addmedia --probe-synthesis '$media->{name}' $distrib_path/media/$media->{mediadir}/release"); if ($media->{updates} eq "yes") { system("LC_ALL=C sudo chroot $chroot_path urpmi.addmedia --update --probe-synthesis \"$media->{name} Updates\" $distrib_path/media/$media->{mediadir}/updates"); } if ($media->{backports} eq "yes") { system("sudo chroot $chroot_path urpmi.addmedia --probe-synthesis '$media->{name} Backports' $distrib_path/media/$media->{mediadir}/backport"); } if ($media->{testing} eq "yes") { system("sudo chroot $chroot_path urpmi.addmedia --probe-synthesis '$media->{name} Testing' $distrib_path/media/$media->{mediadir}/testing"); } } # } else { # system("sudo chroot $chroot_path urpmi.addmedia --distrib $distrib_path --no-md5sum"); # } } sub add_media { my ($name, $path, $update) = @_; print_color("$LOG add media $name from $path", $color); system("sudo chroot $chroot_path mkdir -p /tmp/$name"); system("sudo mount -o bind $path $chroot_path/tmp/$name -o ro"); # check if path contain sub direcroty or not (media/release media/updates etc...) if (-d "$path/release" and -d "$path/updates") { print_color("$LOG This supp media contains release and updates sub-directory", $color); system("LC_ALL=C sudo chroot $chroot_path urpmi.addmedia $name-release /tmp/$name/release"); if ($update =~ /yes/) { system("LC_ALL=C sudo chroot $chroot_path urpmi.addmedia --update '$name Updates' /tmp/$name/updates"); } } else { system("LC_ALL=C sudo chroot $chroot_path urpmi.addmedia $name /tmp/$name"); } push @to_umount, "/tmp/$name"; push @add_media, "$name"; } sub drop_media { my ($name) = @_; print_color("$LOG remove media $name from urpmi config", $color); system("LC_ALL=C sudo chroot $chroot_path urpmi.removemedia $name"); } my $listcopy = "$chroot_path/$isoconf->{tocopy_file}"; sub generate_list_of_rpm_2_copy { my @toget;my @toexclude;my $extra; mini_chroot; foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{external}{name} and add_media($media->{external}{name}, $media->{external}{fullpath_mediadir}, $media->{updates}); foreach my $input (@{$media->{input}}) { $input->{file} or next; -f $input->{file} or print_color("$LOG i can not find the $input->{file}, remove this list or fix...", $error_color); foreach my $tohave (cat_($input->{file})) { if ($tohave =~ /^\s/ || $tohave =~ /^\#/) { next; } elsif ($tohave =~ /^\S*\s\d$/) { my ($section, $poids) = $tohave =~ /^(\S*)\s(\d)/; print "$LOG get packages from $section using weight of $poids\n"; # TOFIX remove META_CLASS to avoid theme pb system("cat $builddir/$isoconf->{mediainfo}{mediainfo_dir}/rpmsrate | grep -v META_CLASS > $tmp_rpmsrate"); # system("cat $tmp_rpmsrate | grep -v theme > $tmp_rpmsrate"); my ($rates, $need_to_copy) = get_list_from_CAT($tmp_rpmsrate, $section, $poids); foreach my $data (@{$need_to_copy}) { push @toget, $data; } next; } else { chomp($tohave); $verbose and print "$LOG $tohave\n"; push @toget, $tohave; next; } } } foreach my $exclude (@{$media->{exclude}}) { $exclude->{file} or next; -f $exclude->{file} or print_color("$LOG i can not find the $exclude->{file}, remove this list or fix...", $error_color); foreach my $totest (cat_($exclude->{file})) { if ($totest =~ /^\s/ || $totest =~ /^\#/) { next; } else { chomp($totest); push @toexclude, $totest; next; } } } # print "$LOG $media, $toget, $toexclude"; create_list($media, \@toget, \@toexclude, $media->{external}{name}, $media->{media_supp}); @toexclude = ""; @toget = ""; } } sub rpmcheck_list { my @toget = cat_($rpmcheck_list_pkg); my $todo = @toget; if ($todo ne "0") { print_color("$LOG calculate deps from the auto-generated rpmcheck list file", $error_color); my $list_filename = "$isoconf->{tocopy_file}_todo_rpmcheck"; my $list = "$chroot_path/urpmq/$list_filename"; my $urpmq_options = "--requires-recursive -c --sources $isoconf->{mediainfo}{urpmi_option}"; my $pkgs; foreach (@toget) { $_ or next ; chomp($_); $pkgs = "$_ " . $pkgs; } my $cmd_do_it = "LC_ALL=C sudo -P chroot $chroot_path urpmq --prefer $DISTRIB,$THEME $urpmq_options --no-suggests"; do_fork($pkgs, $list_filename, $list, $cmd_do_it); } else { print_color("$LOG No rpmcheck error", $error_color); } } sub remove_notpresent { my ($urpmi_error, $wanted) = @_; my @toremove; foreach my $entry (cat_($urpmi_error)) { my ($data, $tor) = $entry =~ /(.*)\s(.*)\n/; $data =~ /-/ and next; $data =~ /^\s/ and next; $tor or next; push @toremove, $tor; } # keep only one line per packages my %already_seen; my @final_list; @already_seen{@toremove} = (); foreach my $object (@{$wanted}) { if (exists $already_seen{$object} and $verbose) { print_color("$LOG i cant find this package: $object, remove it from list", $error_color); } push(@final_list, $object) unless exists $already_seen{$object}; } # prepare the final list my $final_list; foreach (@final_list) { chomp($_); $final_list = $final_list . " $_"; } return $final_list; } sub create_list { my ($media, $wanted, $exclude, $extra, $media_supp) = @_; my $extra_media; # Main media is mandatory for all other media ! # we will use only Main + updates and this media + updates to calculte the list of wanted pkg if ($media->{name} eq "Main") { if ($media->{updates} eq "yes") { #$extra_media = "--media \"Main,Main Updates,$extra,$media_supp\" --sortmedia \"Main Updates,Main\""; $extra_media = "--media \"Main,Main Updates,$extra,$media_supp\""; } else { $extra_media = "--media \"Main,$extra,$media_supp\""; } } else { my $extra2; foreach (@add_media) { chomp($_); $_ and $extra2 = $_ . ',' . $extra2; }; #$extra_media = "--media \"Main,Main Updates,$extra2,$media_supp,$media->{name},$media->{name} Updates\" --sortmedia \"Main Updates,Main,$media->{name} Updates,$media->{name}\""; if ($media->{updates} eq "yes") { $extra_media = "--media \"Main,Main Updates,$extra2,$media_supp,$media->{name},$media->{name} Updates\""; } else { $extra_media = "--media \"Main,$extra2,$media_supp,$media->{name}\""; } } # check the suggests option in the XML configuration my $opt_sug; if ($media->{nosuggests} eq "yes") { $opt_sug = "--no-suggests"; } else { $opt_sug = ""; } # list of wanted RPM per media name my $list_filename = "$isoconf->{tocopy_file}_todo_$media->{name}"; # my $list_filename_update = "$isoconf->{tocopy_file}_todo_$media->{name}_update"; my $list = "$chroot_path/urpmq/$list_filename"; my $list_update = "$chroot_path/urpmq/$list_filename" . "_update"; system("sudo chroot $chroot_path mkdir -m 777 -p urpmq"); # urpmq error file my $urpmq_error = "$chroot_path/$isoconf->{tocopy_file}_urpmi_error_$media->{name}"; my $urpmq_error_update = "$chroot_path/$isoconf->{tocopy_file}_urpmi_error_update_$media->{name}"; # create list of wanted and exlude my $tested_exclude; my $tested_wanted; foreach (@{$wanted}) { $tested_wanted = $tested_wanted . " $_" } foreach (@{$exclude}) { $_ or next; $tested_exclude = $tested_exclude . " --skip $_" }; print_color("$LOG remove unwanted mandriva-release, use $DISTRIB", $color); # urpmq options (with option in the xml file) my $urpmq_options = "--requires-recursive -c --sources $isoconf->{mediainfo}{urpmi_option}"; my $urpmq_extra_opt = "$opt_sug $extra_media"; ### # do the job with update, find missing package from list # only if the media as udpate option enable if ($media->{updates} eq "yes") { my $cmd_find_missing_update = "LC_ALL=C sudo chroot $chroot_path urpmq --update --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_wanted $tested_exclude 2> $urpmq_error_update"; print_color("$LOG first attempt to run urpmq --update on media $media->{name} (to find all packages not availables in update)", $color); system($cmd_find_missing_update); # urpmq error found, so we have to remove some entry in the list of wanted packages my $final_list_update = remove_notpresent($urpmq_error_update, $wanted); my $cmd_do_it_update = "LC_ALL=C sudo -P chroot $chroot_path urpmq --update -a --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_exclude"; do_fork($final_list_update, "$list_filename" . "_update", $list_update, $cmd_do_it_update); } ### # select rpm from all media, find missing from list my $cmd_find_missing = "LC_ALL=C sudo chroot $chroot_path urpmq --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_wanted $tested_exclude 2> $urpmq_error > $list"; print_color("$LOG first attempt to run urpmq on media $media->{name} (to find all packages not availables)", $color); system($cmd_find_missing); # urpmq error found, so we have to remove some entry in the list of wanted packages my $final_list = remove_notpresent($urpmq_error, $wanted); print_color("$LOG launch urpmq on media $media->{name} (to create $list)", $color); my $cmd_do_it = "LC_ALL=C sudo -P chroot $chroot_path urpmq --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_exclude"; do_fork($final_list, $list_filename, $list, $cmd_do_it); } sub do_fork { my ($final_list, $list_filename, $list, $cmd) = @_; my @list_alone = split(/ /, $final_list); my $count = @list_alone; my $status = "0"; my $pm = new Parallel::ForkManager($NB_FORK); print_color("$LOG $count transactions to do ... be patient !!!!", $color); $| = 1; foreach my $pkg (@list_alone) { $pkg or next; $status++; my $pid = $pm->start and next; back_and_print($status, $count); # check an urpmq has been done before in an other media foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; # $dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg" my $previous_urqpm_querie; if (-f "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg") { $previous_urqpm_querie = "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg"; } elsif (-f "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg") { $previous_urqpm_querie = "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg"; } else { $previous_urqpm_querie = "EMPTY"; next; } if ($previous_urqpm_querie =~ /EMPTY/) { $verbose and print_color("$LOG can't found the urpmq transaction for $pkg", $color); } else { $verbose and print_color("$LOG found the urpmq transaction for $pkg, i will use $previous_urqpm_querie, and copy it to $list-$pkg", $color); system("touch $dir_deps/bypass-$pkg"); system("cp -f $previous_urqpm_querie $list-$pkg"); #system("cp -f $dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg $dir_deps/$list_filename-$pkg"); } last; } # check an urpmq has been done before in this media if (-f "$dir_deps/bypass-$pkg") { #my $pkg_found = `grep -P '\/$pkg-' $dir_deps/$list_filename-$pkg`; #$pkg_found = basename($pkg_found); $verbose and print_color("$LOG found the urpmq transaction for $pkg, i will the previous one found in $dir_deps directory", $color); #system("cp -f $dir_deps/$list_filename-$pkg $list-$pkg"); } else { # match the case using --update option my $cmd_do_it = "$cmd $pkg > $list-$pkg"; if ($cmd =~ /\-\-update/) { my @result = `urpmq --update $pkg`; foreach (@result) { # exact match with --update option $_ or next; if ($_ =~ /^$pkg/) { system($cmd_do_it); system("cp -f $list-$pkg $dir_deps/$list_filename-$pkg"); last; } else { $verbose and print "$LOG $pkg is not in update, it's a package that contains the string $pkg!\n"; last } } } else { system($cmd_do_it); system("cp -f $list-$pkg $dir_deps/$list_filename-$pkg"); } } $pm->finish; } print_color("$LOG Waiting for the end of some urpmq command...", $color); $pm->wait_all_children; print_color("$LOG all urpmq command are done...", $color); } sub back_and_print { my ($status, $total) = @_; printf STDERR "%s / %2d \n", $status, $total; print STDERR "\e[A"; } sub concatenate_files { # remove urpmq queries from excluded packages print_color("$LOG remove urpmq queries from removed packages", $color); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; foreach my $exclude (@{$media->{exclude}}) { $exclude->{file} or next; -f $exclude->{file} or print_color("$LOG i can not find the $exclude->{file}, remove this list or fix...", $error_color); foreach my $totest (cat_($exclude->{file})) { chomp($totest); if (! $totest) { next }; system("rm -vf $chroot_path/urpmq/$isoconf->{tocopy_file}_todo_*-$totest"); } } } print_color("$LOG create the final list: $listcopy", $color); system("cat $chroot_path/urpmq/$isoconf->{tocopy_file}_todo_* > $listcopy.tmp"); system("sed -e 's|^\\w*\\s/|/|' $listcopy.tmp > $listcopy"); system("grep '^/' $listcopy > $listcopy.tmp"); system("cat $listcopy.tmp | sort | uniq > $listcopy"); } sub target_size { print_color("$LOG calculate the size of all packages:", $color); my $total;my $pkg_info; foreach my $pkg (cat_($listcopy)) { chomp($pkg); if (-f "$pkg") { # stat is not really good to get size file of linked packages over nfs $pkg_info = stat($pkg); $total = ($total+$pkg_info->size); next; } else { next; } } my $human_total = int(($total/1024)/1024); print_color("$LOG Size of all RPMS: $human_total Mo", $color); my $left = ($isoconf->{iso}{size} - $human_total); # i need to add the install/ directory size if ("$isoconf->{iso}{size}" ge "$human_total") { print_color("$LOG all you data will fit on the CD/DVD, extra size: $left", $color); } else { print_color("$LOG all packages will not fit on $isoconf->{iso}{size}, i need $left Mo", $error_color); } } sub count_nb_to_copy { my ($file) = @_; my $lines = 0; open(FILE, $file) or die "Can't open `$file': $!"; while (sysread FILE, my $buffer, 4096) { if ($buffer =~ /^install/ || $buffer =~ /^\-/ || $buffer =~ /^More/ || $buffer =~ /^Install/ ) { next; } else { $lines += ($buffer =~ tr/\n//); } } close FILE; return $lines; } sub find_source_dest_media { my ($todeal) = @_; # distribution path if ($todeal =~ m!^$distrib_path!) { # find the source media name # f_path will contrib/release, dir_f_path will be contrib my ($f_path) = $todeal =~ /^$distrib_path\/media\/(.*)/; my $dir_f_path = dirname($f_path); foreach my $media (@{$isoconf->{media}{list}}) { # find a mediadir or next one $media->{mediadir} or next; #print "DEBUG media->name: $media->{mediadir} f_path: $f_path\n"; # try to find the source media if ($media->{mediadir} eq $dir_f_path) { #print "DEBUG todeal: $todeal FROM $repo/$based_on/$arch/media/$f_path TO media/$media->{destmedia}\n"; return("$repo/$based_on/$arch/media/$f_path", "media/$media->{destmedia}"); } } next; } else { #external repository #/tmp/u_xos or /tmp/u_xos/release and /tmp/u_xos/updates my ($source_media) = $todeal =~ /\/tmp\/(.*)/; # find the dest media from source media foreach my $media (@{$isoconf->{media}{list}}) { # find a media or next one $media->{external}{name} or next; # print Dumper($media); # print "DEBUG todeal: $todeal DEBUG media->name: $media->{name} source_media: $source_media\n"; # print "$todeal from $source_media | external-fullpath_mediadir: $media->{external}{fullpath_mediadir}/ media/$media->{destmedia}\n"; # print "external-name $media->{external}{name}\n"; if ($source_media eq $media->{name}) { # we have found the media so destmedia is $media->{destmedia} # return the destmedia return($media->{external}{fullpath_mediadir}, "media/$media->{destmedia}"); } elsif ($source_media eq "$media->{name}/release") { return("$media->{external}{fullpath_mediadir}/release", "media/$media->{destmedia}"); } elsif ($source_media eq "$media->{name}/updates") { return("$media->{external}{fullpath_mediadir}/updates", "media/$media->{destmedia}"); } else { print "dont know what to do ....\n"; } next; } } } sub parse_copy_link { my ($wichaction) = @_; my $count;my $action; if ($wichaction =~ /link/) { $action = "ln -sf"; } else { $action = "cp -fa"; } my $total = count_nb_to_copy($listcopy); print_color("$LOG $wichaction all wanted packages to $builddir/XXXX ($total packages)", $color); # chroot packages if (!-f "$chroot_path/$isoconf->{tocopy_file}_TODO_chroot") { exit 1 } foreach my $line (cat_("$chroot_path/$isoconf->{tocopy_file}_TODO_chroot")) { chomp($line); my @list = split(/ /, $line); my $data = $list[$#list]; # if (! -d $data) { print "$data is not a directory !\n"; } foreach my $pkg (@list) { if (-f "$data/$pkg") { #print "$LOG $action $data/$pkg TO1 $builddir/media/main/$pkg\n"; system("$action $data/$pkg $builddir/media/main/$pkg"); next; } elsif (-f $data) { my $package = basename($data); chomp($package); #print "$LOG $action $data TO2 $builddir/media/main/$package\n"; system("$action $data $builddir/media/main/$package"); next; } } } foreach my $tocopy (cat_($listcopy)) { chomp($tocopy); #print "info tocopy $tocopy\n"; my @list = split(/ /, $tocopy); # path final my $data2 = $list[$#list]; my ($path, $pkg) = $tocopy =~ m!^(/.*)/(.*)\.[rpm|]$!; # une ligne avec un mot avant # $pkg or ($path, $pkg) = $tocopy =~ m!^\w+\s(/.*)/(.*)\.rpm$!; my $source_media; my $dest_media; my $package = basename($data2); my $dirn = dirname($data2); if (-f "$chroot_path/$data2" || !-d "$chroot_path/$data2") { chomp($package); ($source_media, $dest_media) = find_source_dest_media($dirn); $dest_media or print "$LOG $package is require/suggest but has no destination, removing it from list\n" and next; #print "parse_copy_link package: $package, sourcemedia: $source_media, dest_media: $dest_media\n"; #print "parse_copy_link $LOG $action $source_media/$package TO3 $builddir/$dest_media/$package\n"; system("$action $source_media/$package $builddir/$dest_media/$package"); next; } else { #print "$chroot_path/$data2 data2 is a directory \n"; ($source_media, $dest_media) = find_source_dest_media($data2); foreach my $pkg2 (@list) { if (-f "$chroot_path/$data2/$pkg2") { print "DATA2/pkg2 i found $chroot_path/$data2/$pkg2 \n"; #print "$LOG $action $source_media/$pkg2 TO4 $builddir/$dest_media/$pkg2\n"; system("$action $source_media/$pkg2 $builddir/$dest_media/$pkg2"); next; } } } $path or next; if (!-d "$chroot_path/$path") { print ""; #print "$chroot_path/$path isnt a directory\n"; } elsif (-f "$chroot_path/$path/$pkg") { #print "$chroot_path/$path/$pkg is a package\n"; ($source_media, $dest_media) = find_source_dest_media($path); $package = basename("chroot_path/$path/$pkg"); chomp($package); #print "$LOG $action $source_media/$package TO5 $builddir/$dest_media/$pkg\n"; system("$action $source_media/$package $builddir/$dest_media/$pkg"); next; } else { print "lost in space !!\n"; } next; } } sub rpmVersionCompare { my ($pkg1, $pkg2) = @_; my ($name_a, $av, $ar) = $pkg1 =~ /^(?:([^:]*):)?([^-]*)(?:-(.*))?$/; my ($name_b, $bv, $br) = $pkg2 =~ /^(?:([^:]*):)?([^-]*)(?:-(.*))?$/; my $rc = 0; if(defined($name_a) && ! defined($name_b)) { return 1; } elsif(!defined($name_a) && defined($name_b)) { return -1; } else { $rc = URPM::rpmvercmp($name_a, $name_b) if (defined($name_a) && defined($name_b)); if ($rc == 0) { $rc = URPM::rpmvercmp($av, $bv); if ($rc == 0) { if(defined($ar) && !defined($br)) { return 1; } elsif(!defined($ar) && defined($br)) { return -1; } elsif (!defined($ar) && !defined($br)) { return 0; } else { return URPM::rpmvercmp($ar, $br); } } else { return $rc; } } else { return $rc; } } } sub remove_all_excluded { # this functions should be available if explicitely needed, it's really too dangerous to remove packages from a list... print_color("$LOG remove all RPM from exclude list", $color); print_color("$LOG WARNING remove needed packages can break dependencies, check rpmcheck messages", $error_color); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; if ($media->{remove_noarch} =~ /yes/) { print_color("$LOG remove noarch from media $media->{name} $builddir/media/$media->{destmedia}/", $color); system("rm -rf $builddir/media/$media->{destmedia}/*.noarch.rpm"); } foreach my $exclude (@{$media->{exclude}}) { $exclude->{file} or next; -f $exclude->{file} or print_color("$LOG i can not find the $exclude->{file}, remove this list or fix...", $error_color); foreach my $totest (cat_($exclude->{file})) { chomp($totest); if (! $totest) { next }; if ($totest =~ /^\s.*/ || $totest =~ /^\#/) { next; } else { my @l = glob("$builddir/media/$media->{destmedia}/$totest*.rpm"); foreach my $file (@l) { $file or next; if (-f $file) { $verbose and print "$LOG $file found, removing this file\n"; system("rm -f $file"); next; } } } next; } next; } } } sub copy_filedeps { print_color("$LOG copy $mediainfo_sourcedir/$isoconf->{mediainfo}{filedeps} in $builddir/$isoconf->{mediainfo}{mediainfo_dir}", $color); -f "$mediainfo_sourcedir/$isoconf->{mediainfo}{filedeps}" or print "$LOG missing filedeps\n"; system("cp -vf $mediainfo_sourcedir/$isoconf->{mediainfo}{filedeps} $builddir/$isoconf->{mediainfo}{mediainfo_dir}/file-deps") == 0 or die "$LOG copy of $mediainfo_sourcedir/$isoconf->{mediainfo}{filedeps} failed: $?\n"; } sub copy_rpmsrate { print_color("$LOG copy $mediainfo_sourcedir/$isoconf->{mediainfo}{rpmsrate} in $builddir/$isoconf->{mediainfo}{mediainfo_dir}", $color); -f "$mediainfo_sourcedir/$isoconf->{mediainfo}{rpmsrate}" or print "$LOG missing rpmsrate\n"; system("cp -vf $mediainfo_sourcedir/$isoconf->{mediainfo}{rpmsrate} $builddir/$isoconf->{mediainfo}{mediainfo_dir}/rpmsrate") == 0 or die "$LOG copy of $mediainfo_sourcedir/$isoconf->{mediainfo}{rpmsrate} failed: $?\n"; } sub copy_compssusers { print_color("$LOG copy $mediainfo_sourcedir/$isoconf->{mediainfo}{compssusers} in $builddir/$isoconf->{mediainfo}{mediainfo_dir}", $color); -f "$mediainfo_sourcedir/$isoconf->{mediainfo}{compssusers}" or print "$LOG missing compssusers\n"; system("cp -vf $mediainfo_sourcedir/$isoconf->{mediainfo}{compssusers} $builddir/$isoconf->{mediainfo}{mediainfo_dir}/compssUsers.pl") == 0 or die "$LOG copy of $mediainfo_sourcedir/$isoconf->{mediainfo}{compssusers} failed: $?\n";; } sub use_genhdlist2 { my $destmedia; foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; if ($destmedia eq $media->{destmedia}) { next } $destmedia = $media->{destmedia}; print_color("$LOG build hdlists and synthesys in $builddir/media/$media->{destmedia}", $color); system("genhdlist2 -v --clean --file-deps $builddir/media/media_info/$isoconf->{medianfo}{filedeps} --media_info-dir $builddir/media/media_info/ $builddir/media/$media->{destmedia}"); system("mv $builddir/media/$destmedia/media_info/hdlist.cz $builddir/media/media_info/hdlist_$media->{name}.cz"); system("mv $builddir/media/$destmedia/media_info/synthesis.hdlist.cz $builddir/media/media_info/synthesis.hdlist_$media->{name}.cz"); # system("genhdlist2 --quiet --clean --no-hdlist --file-deps $builddir/media/media_info/$isoconf->{medianfo}{filedeps} $builddir/media/$media->{destmedia}"); } system("cd $builddir/media/media_info/ && md5sum *.cz > MD5SUM"); } sub rpmcheck_launch { my $options = "-explain -failures"; my $destmedia = "0"; print_color("$LOG launch rpmcheck to check hdlists", $color); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{drop} =~ /yes/ and drop_media("$media->{name}-release"); if ($destmedia eq $media->{destmedia}) { next } $destmedia = $media->{destmedia}; system("cp $builddir/media/$destmedia/media_info/hdlist.cz $builddir/media/media_info/hdlist_$media->{name}.cz"); } my $cmd = "zcat $builddir/media/media_info/hdlist*.cz | rpmcheck $options"; print_color("$LOG $cmd", $color); system("$cmd 2> /dev/null > $file_rpmcheck_log") == 0 or die "$LOG system $cmd failed: $?\n"; open my $LIST_FAILURE, ">$rpmcheck_list_pkg.tmp"; my @list_failure = cat_($file_rpmcheck_log); foreach my $package (@list_failure) { print "$package"; my ($package2) = $package =~ /^\s+(\S*)\s.*/; $package2 and print $LIST_FAILURE "$package2\n"; } close $LIST_FAILURE; system("cat $rpmcheck_list_pkg.tmp | sort | uniq > $rpmcheck_list_pkg"); print_color("$LOG check this file $file_rpmcheck_log", $color); print_color("$LOG check this file $rpmcheck_list_pkg", $color); } sub remove_hdlist { my $destmedia; if ($isoconf->{based_on} gt "2009.1" or $isoconf->{based_on} != "mes5") { print_color("$LOG remove hdlists*.cz from $builddir/media/media_info", $color); system("rm -vf $builddir/media/media_info/hdlist*.cz"); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; if ($destmedia eq $media->{destmedia}) { next } $destmedia = $media->{destmedia}; print_color("$LOG remove hdlists from $builddir/media/$destmedia/media_info", $color); system("rm -vf $builddir/media/$destmedia/media_info/hdlist.cz"); system("cd $builddir/media/$destmedia/media_info ; md5sum *.cz > MD5SUM"); } } } sub copy_all_pubkey { foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; if ($media->{pubkey} eq "") { print_color("$LOG using $media->{file_pubkey}", $color); copy_pubkey($media->{file_pubkey}, "$media->{destmedia}"); } else { # media/ dir will be added in the copy_pubkey functions print_color("$LOG $repo/$based_on/$arch/media/$media->{mediadir}/$media->{pubkey}, $media->{destmedia}", $color); copy_pubkey("$repo/$based_on/$arch/media/$media->{mediadir}/$media->{pubkey}", "$media->{destmedia}"); } next; } } sub list_medias_available { #my @mediacfg = cat_("$repo/$based_on/$arch/$isoconf->{mediainfo_dir}/media.cfg"); print_color("$LOG list all medias availables in $repo/$based_on/$arch repositery", $color); my @mediacfg = cat_("$repo/$based_on/$arch/media/media_info/media.cfg"); my @medias = grep {/^name=/} @mediacfg; foreach my $media (@medias) { $media =~ s/name\=//; $media and print("$media"); } } sub check_repo_hdlist { my $options = "-explain -failures"; print_color("$LOG check media repositery hdlists in $repo/$version/$arch", $color); system("zcat $repo/$version/$arch/media/media_info/hdlist*ain.cz | rpmcheck $options"); } sub umount_all_mountloop { print "$LOG umount all mounted loop dir\n"; foreach (@to_umount) { system("sudo umount $chroot_path/$_"); } } sub get_srpms_list { my $pm = new Parallel::ForkManager(20); my $pkg_name; my @srpms; my @checked; my $already_done; my $tmp_bs = "/tmp/list_BS-$isoconf->{nameid}-$version-$isoconf->{arch}"; system("mkdir $tmp_bs"); system("rm -f $tmp_bs/*"); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{destmedia} or next; foreach (@checked) { $_ or next ; $media->{destmedia} =~ /^$_$/ and $already_done = 1; }; $already_done and next; push @checked, $media->{destmedia}; my @list_pkg = glob("$builddir/media/$media->{destmedia}/*.rpm"); my $list_srpm = "$tmp_bs/list_SRPM-$isoconf->{nameid}-$version-$isoconf->{arch}-$media->{name}-$media->{destmedia}"; my $list_buildreq = "$tmp_bs/list_BUILDREQ-$isoconf->{nameid}-$version-$isoconf->{arch}-$media->{name}-$media->{destmedia}"; my $total = @list_pkg; my $status = "0"; print_color("$LOG i have to find $total SRPMS name in the media media/$media->{destmedia} ... be patient !!!!", $color); foreach my $pkg (@list_pkg) { $status++; my $pid = $pm->start and next; $pkg_name = `rpm -qp --queryformat %{SOURCERPM} $pkg`; # tostore somewhere # $media->{name} $pkg_name system("sudo chroot $chroot_path echo '$pkg_name' > $list_srpm-$pkg_name"); back_and_print($status, $total); $pm->finish; $pm->wait_all_children; } system("cat $tmp_bs/list_SRPM-* | sort | uniq > $list_srpm"); my @list_srpm = cat_($list_srpm); $total = @list_srpm; $status = "0"; print_color("$LOG get the buildrequires from SRPMS", $color); foreach my $srpm (@list_srpm) { $srpm or next; $status++; chomp($srpm); my $pid = $pm->start and next; my $pkg = `basename $srpm .src.rpm`; my $SRPMS_PATH; # find the package in an official sub dir my @struct = qw(release updates backport testing); back_and_print($status, $total); my @data; my $q; foreach (@struct) { #print "$repo/$based_on/SRPMS/$media->{mediadir}/$_/$srpm"; if (-f "$repo/$based_on/SRPMS/$media->{mediadir}/$_/$srpm") { $SRPMS_PATH="$repo/$based_on/SRPMS/$media->{mediadir}/$_/"; my $media="--sortmedia \"Main Updates,Main,Contrib Updates,Contrib\""; system("LC_ALL=C urpmq --use-distrib $repo/$based_on/$arch $media --sources --requires-recursive --prefer $DISTRIB,$THEME,kernel-source-dekstop --no-suggests $SRPMS_PATH/$srpm > $list_buildreq-$srpm"); last; } else { #print "$repo/$based_on/SRPMS/$media->{mediadir}/$_/$srpm doesnt exist \n"; next; } } $pm->finish; $pm->wait_all_children; } system("cat $tmp_bs/list_BUILDREQ-* | sort | uniq > $list_buildreq"); system("cp $list_buildreq $chroot_path/list_ALL_BUILDREQ"); system("cp $list_srpm $chroot_path/list_ALL_SRPM"); # system("rm -f $tmp_bs/list_BUILDREQ-* $tmp_bs/list_SRPM-*"); } } sub extra_filter { my ($filter) = @_; print_color("$LOG compare versions of $filter and keep the latest one", $color); my @l; my $keeped; foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{destmedia} or next; @l = glob("$builddir/media/$media->{destmedia}/$filter-*.rpm"); my $l = @l; $l eq "0" and last; foreach (@l) { my $pkg = basename($_); # check it will be a number after the filter name $pkg =~ /$filter\-\d.*/ or next; $keeped or $keeped = $pkg; $pkg ne $keeped and print "$LOG COMPARE $pkg and $keeped:\n" or next; my $r = rpmVersionCompare($pkg, $keeped); if ($r eq "-1") { $keeped = $keeped; print "i will keep $keeped\n"; system("rm -vf $builddir/media/$media->{destmedia}/$pkg"); next; } elsif ($r eq "1") { system("rm -vf $builddir/media/$media->{destmedia}/$keeped"); $keeped = $pkg; print "i will keep $keeped\n"; next; }; } } } sub solve_doble_issue { my $pkg_name; my $pkg_version; my @list_to_filter; my @todeal; my %already_seen; my @final_list; my $keeped; # 20 should not cause any trouble... NB_FORK is too small... my $pm = new Parallel::ForkManager(20); my @checked; my $already_done; foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{destmedia} or next; foreach (@checked) { $_ or next; $_ =~ /^\#/ and next ; $media->{destmedia} =~ /^$_$/ and $already_done = 1; }; $already_done and next; push @checked, $media->{destmedia}; # get all rpm from the media dir my @list_pkg = glob("$builddir/media/$media->{destmedia}/*.rpm"); my $total = @list_pkg; my $status = "0"; print_color("$LOG i have to check $total doble in the media media/$media->{destmedia} ... be patient !!!!", $color); # parse all files in the media foreach my $pkg (@list_pkg) { $status++; # get the rpm name my $pid = $pm->start and next; $pkg_name = `rpm -qp --queryformat %{name} $pkg`; my @l = glob("$builddir/media/$media->{destmedia}/$pkg_name*.rpm"); my $l = @l; $l eq "0" and last; foreach (@l) { #print_color("$LOG compare versions of $filter and keep the latest one", $color); $_ or next; my $found = 0; my $pkg = basename($_); foreach (cat_($isoconf->{mediainfo}{filter})) { if ($_ =~ /^$pkg_name/) { $found = 1; } #print "- skipping $_\n (from $isoconf->{mediainfo}{filter} file)" and $found = 1; } } #if ($_ =~ /kernel/) { print "- $pkg_name is a package relative to a kernel ! will be treated later\n"; } $found eq "1" and next; $pkg =~ /^$pkg_name\-\d.*/ or next; # bypass special case like libwnck-1_22 $pkg =~ /^$pkg_name\-\d\_.*/ and print "- $pkg comparaison skipped... (bcd regexp)\n" and next; # check it will be a number after the filter name $keeped or $keeped = $pkg; #print "$LOG COMPARE $pkg and $keeped: \n"; my $r = rpmVersionCompare($pkg, $keeped); if ($r eq "-1") { $keeped = $keeped; $verbose and print "i will keep $keeped and erase $builddir/media/$media->{destmedia}/$pkg\n"; system("rm -f $builddir/media/$media->{destmedia}/$pkg"); push @list_of_removed, "$builddir/media/$media->{destmedia}/$pkg"; } elsif ($r eq "1") { system("rm -f $builddir/media/$media->{destmedia}/$keeped"); $keeped = $pkg; $verbose and print "i will keep $pkg and remove $builddir/media/$media->{destmedia}/$keeped\n"; push @list_of_removed, "$builddir/media/$media->{destmedia}/$keeped"; }; } back_and_print($status, $total); $pm->finish; print_color("$LOG Waiting for the end of some system command...", $color); $pm->wait_all_children; } } } sub find_all_kernel { print_color("$LOG kernel comparaison", $color); foreach my $media (@{$isoconf->{media}{list}}) { $media->{name} or next; $media->{destmedia} or next; my @l = glob("$builddir/media/$media->{destmedia}/kernel-*.rpm"); my $l = @l; $l eq "0" and last; foreach my $k (@l) { my ($kname, $kversion) = $k =~ m/kernel-([\w]+(?:\-[\w]+){0,3})-([0-9]+(?:\.[0-9]+){2,3})/; $kname or next; #print_color("$LOG i will compare all versions of kernel-$kname and keep the latest one", $color); extra_filter("kernel-$kname"); next; } } } sub process_list_link { my ($option) = @_; concatenate_files; # link or copy if ($option =~ /copy/) { parse_copy_link("copy"); } else { parse_copy_link("link"); } # remove packages listed in exclude list remove_all_excluded; if ($option =~ /nodoble/) { print_color("$LOG skipping doble\n", $color); } else { solve_doble_issue; } find_all_kernel; copy_filedeps; copy_rpmsrate; copy_compssusers; use_gendistrib; ## use_genhdlist2; } sub clean_urpmq_log { my ($option) = @_; if ($option =~ /clean/) { print_color("$LOG i will remove all previous urpmq log", $color); system("rm -rf $dir_deps"); } } sub main_media { my ($option) = @_; if ($option =~ /verbose/) { $verbose = "1"; } list_media; create_media; create_mediacfg; copy_all_pubkey; create_needed_dir($chroot_path); print_color("$LOG erase previous bypasss file", $color); system("rm -f $dir_deps/bypass-*"); copy_rpmsrate; generate_list_of_rpm_2_copy; process_list_link($option); # target_size; # first rpmcheck launch rpmcheck_launch; rpmcheck_list; process_list_link($option); # second launch of rpmcheck (should have fix a lot of missing deps) rpmcheck_launch; umount_all_mountloop; #remove_hdlist; show_size; my $l = @list_of_removed; if ($l ne "0") { print_color("$LOG List of package removed, if you dont want to remove them, just add them in the file $isoconf->{mediainfo}{filter}:", $color); foreach (@list_of_removed) { print "$_\n"; } } } 1;