- update isos configuration files

- switch to trunk version of bcd
- update Media.pm

5 files changed, 113 insertions, 49 deletions

diff --git a/BCD/Bcd.pm b/BCD/Bcd.pm
index 98a68e1..ce0bffa 100644
--- a/BCD/Bcd.pm
+++ b/BCD/Bcd.pm
@@ -63,6 +63,7 @@ sub main_bcd() {
 	mediarepo => \&list_medias_available,
 	checkrepo => \&check_repo_hdlist,
         rpmcheck => \&rpmcheck_launch,
+	smartcheck => \&smartcheck_launch,
 	doble => \&solve_doble_issue,
 	kernel => \&find_all_kernel,
 	resign => \&resign_media,
diff --git a/BCD/Common.pm b/BCD/Common.pm
index 90cfceb..126c207 100644
--- a/BCD/Common.pm
+++ b/BCD/Common.pm
@@ -57,7 +57,8 @@ $version or print_color("You must define the <mediainfo><version>", $color) and
 $repo or print_color("You must define the <repo><path> tag",$color) and exit 1;
 $based_on or print_color("You must define the <based_on> tag",$color) and exit 1;
 $isoconf->{chroot_media} or print_color("Now you must define <chroot_media> in the conf.xml file",$color) and exit 1;
-$isoconf->{chroot_media_exclude} or print_color("Now you must define <chroot_media_exclude> in the conf.xml file",$color) and exit 1; 
+$isoconf->{chroot_media_exclude} or print_color("Now you must define <chroot_media_exclude> in the conf.xml file",$color) and exit 1;
+
 
 # password file
 #"/home/plop/.signature.gpg";
diff --git a/BCD/Genisoimage.pm b/BCD/Genisoimage.pm
index 2f031d7..4a5cc42 100644
--- a/BCD/Genisoimage.pm
+++ b/BCD/Genisoimage.pm
@@ -56,6 +56,7 @@ sub create_idx {
 	open my $idx_file, ">$idx";
 	foreach (keys %hrpms) {
 		$_ or next;
+		print $_;
 		my @table = @{ $hrpms{$_} };
 #                print "$isoconf->{iso}{header}{volumeid} $_-$table[0]-$table[1]\n";
 		print $idx_file "$isoconf->{iso}{header}{volumeid} $_-$table[0]-$table[1]\n";
@@ -106,7 +107,7 @@ sub create_md5 {
 sub isohybrid {
 	print_color("$LOG isohybrid the ISO", $color);
 	my $ISOFILE = "$isodir/$isoname.iso";
-	#system("isohybrid $iso->{options}{isohybrid} $ISOFILE") == 0 or die "$LOG system isohybrid failed: $?\n";
+#	system("isohybrid $iso->{options}{isohybrid} $ISOFILE") == 0 or die "$LOG system isohybrid failed: $?\n";
 }
 
 sub copy_textfiles {
@@ -124,7 +125,7 @@ sub main_iso {
 		print_color("$LOG skipping ISO creation", $color);
 	} else {
 		create_iso;
-#		isohybrid;
+		isohybrid;
 		create_md5;
 	}
 
diff --git a/BCD/Media.pm b/BCD/Media.pm
index 9f603b4..ab9d309 100644
--- a/BCD/Media.pm
+++ b/BCD/Media.pm
@@ -14,8 +14,8 @@ use Data::Dumper;
 use BCD::Common qw(:DEFAULT $isoconf $wd $name $arch $version $based_on $repo $builddir $distrib_path $error_color $DISTRIB $THEME $tmp_rpmsrate $NB_FORK $dir_deps @list_of_removed);
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(main_media list_media use_gendistrib list_medias_available check_repo_hdlist rpmcheck_launch solve_doble_issue clean_urpmq_log get_srpms_list find_all_kernel, create_mediacfg parse_synthesis);
-our @EXPORT_OK = qw($rpmcheck_list_pkg $file_rpmcheck_log);
+our @EXPORT = qw(main_media list_media use_gendistrib list_medias_available check_repo_hdlist rpmcheck_launch solve_doble_issue clean_urpmq_log get_srpms_list find_all_kernel, create_mediacfg parse_synthesis smartcheck_launch);
+our @EXPORT_OK = qw($rpmcheck_list_pkg $file_rpmcheck_log $smartcheck_list_pkg $smartcheck_dir);
 
 my $verbose;
 my $LOG="MEDIA -";
@@ -35,6 +35,9 @@ if (defined($isoconf->{mediainfo}{fullpath})) {
 my $chroot_path = "/tmp/$isoconf->{nameid}-$version-$isoconf->{arch}";
 our $rpmcheck_list_pkg = "/tmp/rpmcheck_list_pkg_to_add_$isoconf->{nameid}-$version-$isoconf->{arch}";
 our $file_rpmcheck_log = "/tmp/rpmcheck_failure_$isoconf->{nameid}-$version-$isoconf->{arch}";
+our $smartcheck_list_pkg = "/tmp/smartcheck_list_pkg_to_add_$isoconf->{nameid}-$version-$isoconf->{arch}";
+our $file_smartcheck_log = "/tmp/smartcheck_failure_$isoconf->{nameid}-$version-$isoconf->{arch}";
+our $smartcheck_dir = "/tmp/smartcheck_dir_$isoconf->{nameid}-$version-$isoconf->{arch}";
 
 sub create_media {
     print_color("$LOG Create the media dir in $builddir", $color);
@@ -108,7 +111,7 @@ sub use_gendistrib {
 		print_color("$LOG < 2009.1 or mes5 detected", $color);
 		print_color("$LOG move hdlists and synthesys from $builddir/media/$destmedia/media_info to $builddir/media/media_info/", $color);
 		system("rm -vf $builddir/media/media_info/*_$media->{name}.cz");
-		system("rm -vf $builddir/media/media_info/*_main.cz");
+#		system("rm -vf $builddir/media/media_info/*_main.cz");
 		system("rm -vf $builddir/media/media_info/pubkey_$media->{name}");
 		system("cp -avf $builddir/media/$destmedia/media_info/hdlist.cz $builddir/media/media_info/hdlist_$media->{name}.cz");
 		system("cp -avf $builddir/media/$destmedia/media_info/pubkey $builddir/media/media_info/pubkey_$media->{name}");
@@ -248,31 +251,23 @@ sub mini_chroot {
 	# it is usefull to use a mini chroot for add remove media, we have to keep the list of installed packages to copy them on ISO
 	print_color("$LOG create a mini chroot to run urpmi", $color);
 	# store all file needed to be installed on the system
-        my $cmd = "LC_ALL=C sudo urpmi --quiet --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root $chroot_path --media '$isoconf->{chroot_media}' --excludemedia '$isoconf->{chroot_media_exclude}'";
+	my $cmd = "LC_ALL=C sudo urpmi --quiet --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root $chroot_path --media '$isoconf->{chroot_media}' --excludemedia '$isoconf->{chroot_media_exclude}'";
 	#  sudo urpmi --no-verify-rpm --auto --urpmi-root /tmp/mk --use-distrib /mnt/BIG/dis/2009.0/i586/ basesystem-minimal urpmi --test
 	#
-#J	print_color("---DEBUG INSTALLABLE_START $LOG test if the chroot is installable\n LC_ALL=C sudo urpmi --test --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root /tmp/$isoconf->{nameid}-$version-$isoconf->{arch} --media 'Main,Main Updates' --excludemedia 'Main Testing,Main Backports' urpmi basesystem-minimal bash basesystem > $chroot_path/$isoconf->{tocopy_file}_TODO_chroot", $color);
-	system("LC_ALL=C sudo urpmi --test --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root /tmp/$isoconf->{nameid}-$version-$isoconf->{arch} --media 'Main,Main Updates' --excludemedia 'Main Testing,Main Backports' urpmi basesystem-minimal bash basesystem > $chroot_path/$isoconf->{tocopy_file}_TODO_chroot");
-#	print_color("---DEBUG INSTALLABLE_STOP the TEST");
+	system("LC_ALL=C sudo urpmi --test --no-suggests --auto --no-verify-rpm --use-distrib $repo/$based_on/$arch --urpmi-root $chroot_path --media '$isoconf->{chroot_media}' --excludemedia '$isoconf->{chroot_media_exclude}' urpmi basesystem-minimal bash basesystem > $chroot_path/$isoconf->{tocopy_file}_TODO_chroot");
 	# install all those file to be able to use urpmi.XXXXXX
-#	print_color("---DEBUG INSTALL_CHROOT_START now create the chroot:\n $cmd --prefer $DISTRIB,$THEME urpmi bash", $color);
-	system("$cmd --prefer $DISTRIB,$THEME urpmi bash");
-#	print_color("---DEBUG INSTALL_CHROOT_STOP", $color);
-	if (! -e "$chroot_path/bin/mkdir") {
-		print_color("$LOG ERROR, mkdir is not available in $chroot_path", $color);
-		exit 1;
-	}
+	system("LC_ALL=C $cmd --prefer $DISTRIB,$THEME urpmi rootfiles rpm-helper mageia-release-common");
         system("sudo chroot $chroot_path mkdir -p $distrib_path");
 	system("sudo mount -o bind $repo/$based_on/$arch $chroot_path/$distrib_path -o ro");
 	push @to_umount, $distrib_path;
-        print_color("$LOG Add a profile info", $color);
-        my $INFOFILE = "$chroot_path/etc/profile.d/99info.sh";
-        open my $INFO, ">$INFOFILE";
-        print $INFO "#!/bin/sh\n";
-        print $INFO "DIS=`cat /etc/product.id`\n";
+	print_color("$LOG Add a profile info", $color);
+	my $INFOFILE = "$chroot_path/etc/profile.d/99info.sh";
+	open my $INFO, ">$INFOFILE";
+	print $INFO "#!/bin/sh\n";
+	print $INFO "DIS=`cat /etc/product.id`\n";
         print $INFO "export PS1=\"\\n\$DIS\\n [ CHROOT BY BCD \\u@\\h \\W]\$ \"";
-        close $INFO;
-        system("chmod 755 $INFOFILE");
+	close $INFO;
+	system("chmod 755 $INFOFILE");
 	system("sudo chroot $chroot_path urpmi.removemedia -a");
 	foreach my $media (@{$isoconf->{media}{list}}) {
 		$media->{name} or next;
@@ -288,6 +283,9 @@ sub mini_chroot {
 			system("sudo chroot $chroot_path urpmi.addmedia --probe-synthesis '$media->{name} Testing' $distrib_path/media/$media->{mediadir}/testing");
 		}
 	}
+#        } else {
+#            system("sudo chroot $chroot_path urpmi.addmedia --distrib $distrib_path --no-md5sum");
+#	}
 }
 
 sub add_media {
@@ -386,6 +384,23 @@ sub rpmcheck_list {
     }
 }
 
+sub smartcheck_list {
+    my @toget = cat_($smartcheck_list_pkg);
+    my $todo = @toget;
+    if ($todo ne "0") {
+	print_color("$LOG calculate deps from the auto-generated smart list file", $error_color);
+	    my $list_filename = "$isoconf->{tocopy_file}_todo_smartcheck";
+	    my $list = "$chroot_path/urpmq/$list_filename";
+	    my $urpmq_options = "--requires-recursive -c --sources $isoconf->{mediainfo}{urpmi_option}";
+	    my $pkgs;
+	    foreach (@toget) { $_ or next ; chomp($_); $pkgs = "$_ " . $pkgs; }
+	    my $cmd_do_it = "LC_ALL=C sudo -P chroot $chroot_path urpmq --prefer $DISTRIB,$THEME $urpmq_options --no-suggests";
+	    do_fork($pkgs, $list_filename, $list, $cmd_do_it);
+    } else {
+	print_color("$LOG No smart check error", $error_color);
+    }
+}
+
 sub remove_notpresent {
     my ($urpmi_error, $wanted) = @_;
     my @toremove;
@@ -431,15 +446,18 @@ sub create_list {
 	if ($media->{updates} eq "yes") {
 		$extra_media = "--media \"$isoconf->{media}{first_media} $isoconf->{media}{update_prefix},$extra2,$media_supp,$media->{name},$media->{name} $isoconf->{media}{update_prefix}\"";
 	} else {
-		$extra_media = "--media \"$isoconf->{media}{first_media},$extra2,$media_supp,$media->{name}\"";
+                $extra_media = "--media \"$isoconf->{media}{first_media},$extra2,$media_supp,$media->{name}\"";
 	}
     }
 
     # check the suggests option in the XML configuration
-    my $opt_sug;
+    my $opt_sug; my $opt_srpm;
     if ($media->{nosuggests} eq "yes") {
 	$opt_sug = "--no-suggests";
     } else { $opt_sug = ""; } 
+    if ($media->{nsrpm} eq "yes") {
+	$opt_srpm = "--src";
+    } else { $opt_srpm = ""; } 
     # list of wanted RPM per media name
     my $list_filename = "$isoconf->{tocopy_file}_todo_$media->{name}";
 #    my $list_filename_update = "$isoconf->{tocopy_file}_todo_$media->{name}_update";
@@ -461,20 +479,20 @@ sub create_list {
     print_color("$LOG remove unwanted mandriva-release, use $DISTRIB", $color);
     # urpmq options (with option in the xml file)
     my $urpmq_options = "--requires-recursive -c --sources $isoconf->{mediainfo}{urpmi_option}";
-    my $urpmq_extra_opt = "$opt_sug $extra_media";
+    my $urpmq_extra_opt = "$opt_sug $opt_srpm $extra_media";
 
     ###
     # do the job with update, find missing package from list
     # only if the media as udpate option enable
-    if ($media->{updates} eq "yes") {
-	    my $cmd_find_missing_update = "LC_ALL=C sudo chroot $chroot_path urpmq --update --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_wanted $tested_exclude 2> $urpmq_error_update";
-	    print_color("$LOG first attempt to run urpmq --update on media $media->{name} (to find all packages not availables in update)", $color);
-	    system($cmd_find_missing_update);
-	    # urpmq error found, so we have to remove some entry in the list of wanted packages
-	    my $final_list_update = remove_notpresent($urpmq_error_update, $wanted);
-	    my $cmd_do_it_update = "LC_ALL=C sudo -P chroot $chroot_path urpmq --update -a --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_exclude";
-	    do_fork($final_list_update, "$list_filename" . "_update", $list_update, $cmd_do_it_update);
-    }
+##    if ($media->{updates} eq "yes") {
+##	    my $cmd_find_missing_update = "LC_ALL=C sudo chroot $chroot_path urpmq --update --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_wanted $tested_exclude 2> $urpmq_error_update";
+##	    print_color("$LOG first attempt to run urpmq --update on media $media->{name} (to find all packages not availables in update)", $color);
+##	    system($cmd_find_missing_update);
+##	    # urpmq error found, so we have to remove some entry in the list of wanted packages
+##	    my $final_list_update = remove_notpresent($urpmq_error_update, $wanted);
+##	    my $cmd_do_it_update = "LC_ALL=C sudo -P chroot $chroot_path urpmq --update -a --prefer $DISTRIB,$THEME $urpmq_options $urpmq_extra_opt $tested_exclude";
+##	    do_fork($final_list_update, "$list_filename" . "_update", $list_update, $cmd_do_it_update);
+##    }
 
     ###
     # select rpm from all media, find missing from list
@@ -498,6 +516,8 @@ sub do_fork {
     print_color("$LOG $count transactions to do ... be patient !!!!", $color);
     $| = 1;
     foreach my $pkg (@list_alone) {
+	my $pkg_name_ok = $pkg;
+       	$pkg_name_ok =~ s/[\(|\)|\/]/_/g;
 	$pkg or next;
 	$status++;
 	my $pid = $pm->start and next; 
@@ -507,35 +527,37 @@ sub do_fork {
 		$media->{name} or next;
 		# $dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg"
 		my $previous_urqpm_querie;
-		if (-f "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg") {
-			$previous_urqpm_querie = "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg";
-		} elsif (-f "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg") {
-			$previous_urqpm_querie = "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg";
+		if (-f "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg_name_ok") {
+			$previous_urqpm_querie = "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}_update-$pkg_name_ok";
+		} elsif (-f "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg_name_ok") {
+			$previous_urqpm_querie = "$dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg_name_ok";
 		} else {
 			$previous_urqpm_querie = "EMPTY";
 			next;
 		}
 		if ($previous_urqpm_querie =~ /EMPTY/) {
                         $verbose and  print_color("$LOG can't found the urpmq transaction for $pkg", $color);
+		} elsif ($media->{srpm} eq "yes") {
+                        $verbose and  print_color("$LOG srpm detected, dont use previous urpmq transaction for $pkg", $color);
 		} else {
-			$verbose and  print_color("$LOG found the urpmq transaction for $pkg, i will use $previous_urqpm_querie, and copy it to $list-$pkg", $color);
-			system("touch $dir_deps/bypass-$pkg");
-			system("cp -f $previous_urqpm_querie $list-$pkg");
+			$verbose and  print_color("$LOG found the urpmq transaction for $pkg, i will use $previous_urqpm_querie, and copy it to $list-$pkg_name_ok", $color);
+			system("touch $dir_deps/bypass-$pkg_name_ok");
+			system("cp -f $previous_urqpm_querie $list-$pkg_name_ok");
 			#system("cp -f $dir_deps/$isoconf->{tocopy_file}_todo_$media->{name}-$pkg $dir_deps/$list_filename-$pkg");
 		}
 		last;
 	}
 	# check an urpmq has been done before in this media
-	if (-f "$dir_deps/bypass-$pkg") {
+	if (-f "$dir_deps/bypass-$pkg_name_ok") {
 		#my $pkg_found = `grep -P '\/$pkg-' $dir_deps/$list_filename-$pkg`;
 		#$pkg_found = basename($pkg_found);
 		$verbose and print_color("$LOG found the urpmq transaction for $pkg, i will the previous one found in  $dir_deps directory", $color);
 		#system("cp -f $dir_deps/$list_filename-$pkg $list-$pkg");
 	} else {
 	    # match the case using --update option
-		my $cmd_do_it = "$cmd $pkg > $list-$pkg";
+		my $cmd_do_it = "$cmd '$pkg' > $list-$pkg_name_ok";
 		if ($cmd =~ /\-\-update/) {
-			my @result = `urpmq --update $pkg`;
+			my @result = `urpmq --update "$pkg"`;
 			foreach (@result) {
 			    # exact match with --update option
 				$_ or next;
@@ -547,7 +569,7 @@ sub do_fork {
 			}
 		} else {
 	    		system($cmd_do_it);
-			system("cp -f $list-$pkg $dir_deps/$list_filename-$pkg");
+			system("cp -f $list-$pkg_name_ok $dir_deps/$list_filename-$pkg_name_ok");
 		}
     	}
 	$pm->finish;
@@ -881,6 +903,42 @@ sub rpmcheck_launch {
     print_color("$LOG check this file $rpmcheck_list_pkg", $color);
 }
 
+sub smartcheck_launch {
+    my $options = "-o sync-urpmi-medialist=no --data-dir $smartcheck_dir";
+    system("rm -rf $smartcheck_dir $smartcheck_list_pkg");
+    my $destmedia = "0";
+    my $MEDIAS; my $MEDIASWITHCOMMA;
+    print_color("$LOG launch smart to check synthesys", $color);
+    foreach my $media (@{$isoconf->{media}{list}}) {
+	$media->{name} or next;
+	$media->{drop} =~ /yes/ and drop_media("$media->{name}-release");
+	if ($destmedia eq $media->{destmedia}) { next }
+	$destmedia = $media->{destmedia};
+	$MEDIAS = $MEDIAS . " " . $media->{name};
+	$MEDIASWITHCOMMA = $media->{name} . "," . $MEDIASWITHCOMMA;
+	my $cmd = "smart channel --yes $options --add $media->{name} type=urpmi baseurl=$builddir/media/$media->{destmedia}/ hdlurl=media_info/synthesis.hdlist.cz";
+	system($cmd);
+    }
+    system("smart update $options $MEDIAS");
+    chop($MEDIASWITHCOMMA);
+    my $cmd = "smart check $options --channels=$MEDIASWITHCOMMA 2>&1 | egrep -v '(Loading cache|Updating cache)' | awk -F 'requires' '{print \$2}' | sort | uniq  >> $smartcheck_list_pkg";
+    system ("du $smartcheck_list_pkg");
+    print_color("$LOG $cmd", $color);
+    system("$cmd > $file_smartcheck_log") == 0 or die "$LOG system $cmd failed: $?\n";
+    open my $LIST_FAILURE, ">$smartcheck_list_pkg.tmp";
+    my @list_failure = cat_($file_smartcheck_log);
+    foreach my $package (@list_failure) {
+	print "$package";
+	my ($package2) = $package =~ /^\s+(\S*)\s.*/;
+	$package2 and print $LIST_FAILURE "$package2\n";
+    }
+    close $LIST_FAILURE;
+    system("cat $smartcheck_list_pkg.tmp | sort | uniq > $smartcheck_list_pkg");
+    print_color("$LOG check this file $file_smartcheck_log", $color);
+    print_color("$LOG check this file $smartcheck_list_pkg", $color);
+}
+
+
 sub remove_hdlist {
 	my $destmedia;
 	if ($isoconf->{based_on} gt "2009.1" or $isoconf->{based_on} != "mes5") {
@@ -1161,9 +1219,12 @@ sub main_media {
     # first rpmcheck launch
     rpmcheck_launch;
     rpmcheck_list;
+#    smartcheck_launch;
+#    smartcheck_list;
     process_list_link($option);
-    # second launch of rpmcheck (should have fix a lot of missing deps)
+    # second launch of rpmcheck/smartcheck (should have fix a lot of missing deps)
     rpmcheck_launch;
+#    smartcheck_launch;
     umount_all_mountloop;
     #remove_hdlist;
     show_size;
diff --git a/BCD/Web.pm b/BCD/Web.pm
index bf0aff5..70cf501 100644
--- a/BCD/Web.pm
+++ b/BCD/Web.pm
@@ -101,7 +101,7 @@ h3 {
 	<BODY>
 	";
 	print $B "<ul>\n";
-	print $B "<li>Date of the synchronisation to the cauldron repository: <b>$date_repo</b></li>\n";
+	print $B "<li>Date of the synchronisation to the cooker repository: <b>$date_repo</b></li>\n";
 	print $B "<li>Configuration file used: <a href='$conf_file'>$conf_file</a></li>\n";
 	print $B "<li>Architecture: <b>$arch</b></li>\n";
 	print $B "<li>Distribution based on: <b>$based_on</b></li>\n";