add chroot rebuild with iurt2

1 files changed, 430 insertions, 0 deletions

diff --git a/iurt2 b/iurt2
new file mode 100755
index 0000000..f64ae5b
--- /dev/null
+++ b/iurt2
@@ -0,0 +1,430 @@
+#!/usr/bin/perl
+#
+# Copyright (C) 2005 Mandrakesoft
+# Copyright (C) 2005 Mandriva
+# 
+# Author: Florent Villard <warly@mandraesoft.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# compare and rebuild packages on different architecture
+#
+use strict;
+use Hdlist;
+use Data::Dumper;
+use URPM;
+use MIME::Words qw(encode_mimewords);
+use Fcntl ':flock';
+my $home = '/home/mandrake/';
+my $cache_home = "$home/.bugs/";
+my @supported_arch = ('i586', 'x86_64');
+my $upload = "$home/uploads/";
+my $local_home = '/export/home/mandrake';
+my $base_dir = $upload;
+my $check_queue = 0;
+if (@ARGV[0] eq '--check_queue') { 
+	$check_queue = 1;
+	shift @ARGV;
+	$base_dir = shift @ARGV;
+}
+
+my $distro_version = shift @ARGV;
+my $distro_tag = $distro_version;
+$distro_tag =~ s,/,-,g;
+my $cache = "$cache_home/iurt.$distro_tag.cache";
+print "Loading cache file $cache\n";
+eval { require $cache };
+$::rpm_srpm ||= {};
+$::failure ||= {};
+$::queue ||= {};
+$::warning ||= {};
+my $my_arch = shift @ARGV;
+my $media = shift @ARGV;
+my @special_srpm_dir = @ARGV;
+
+my $unwanted_packages = '^monotone-';
+
+$my_arch or usage();
+if ($check_queue) {
+	-d $base_dir or usage("$base_dir does not exist");
+} else {
+    $base_dir .= $distro_version;
+    $base_dir =~ s/community//g;
+    if ($distro_version ne 'cooker') {
+        if ($media ne 'main') {
+	    $base_dir .= "/$media"
+        }
+    } elsif ($media eq 'contrib') {
+	$base_dir = "$upload/contrib"
+    }
+    -d $base_dir or usage("$base_dir does not exist")
+}
+
+my $pidfile = "$cache_home/iurt.$distro_tag.$my_arch.pid";
+my (@stat) = stat $pidfile;
+-f $pidfile and do {
+	open PID, $pidfile;
+	my $pid = <PID>;
+	close PID;
+	if ($pid && getpgrp $pid != -1) {
+		if ($stat[9] < time - 36000) {
+			print "An other iurt pid $pid is running for a very long time, killing it\n";
+			my $i;
+			while ($i < 5 && getpgrp $pid != -1) {
+				kill 9, $pid;
+				$i++;
+				sleep 1
+			}
+		} else  {
+			print "An other iurt is running for $my_arch, pid $pid\n";
+			exit
+		}
+	} else {
+		print "A previous iurt for $my_arch seems dead, cleaning.\n";
+		unlink $pidfile
+	}
+};
+
+open PID, ">$pidfile";
+print PID $$;
+close PID;
+
+my %big;
+my %srpm_version;
+my @wrong_rpm;
+foreach my $arch (@supported_arch) {
+	my $rpms_dir = "/mnt/BIG/dis/$distro_version/$arch/media/$media/";
+	print "Checking current packages in $rpms_dir\n";
+	opendir my $rpmdir, $rpms_dir; 
+	foreach my $rpm (readdir $rpmdir) {
+		my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm) or next;
+		$big{$rpm} = 1;
+		$::queue->{$srpm}{$arch} = 1;
+		$::queue->{$srpm}{$rarch} = 1;
+		check_version($srpm)
+	}
+	closedir $rpmdir
+}
+
+if ($check_queue) {
+	my $dir = "$base_dir/build/RPMS/";
+	opendir my $rpmdir, $dir;
+
+	print "Checking upload queue in $dir\n";
+	foreach my $rpm (readdir $rpmdir) {
+		$rpm =~ /\.rpm$/ or next;
+		if ($big{$rpm}) {
+			print "Cleaning $rpm\n";
+			unlink "$dir/$rpm";
+			next
+		}
+		my ($arch, $srpm) = update_srpm($dir, $rpm) or next;
+		$::queue->{$srpm}{$arch} = 1;
+		check_version($srpm)
+	}
+	closedir $rpmdir;
+
+}
+
+my %maint;
+
+my @todo;
+-d "$base_dir/build/$my_arch" or mkdir "$base_dir/build/$my_arch";
+push @special_srpm_dir, "$base_dir/build/SRPMS/" if $check_queue;
+my $clean;
+my %rep;
+my %done_rpm;
+#
+# FIXME all the rep but the first one are cleaned
+#
+foreach my $dir ("/mnt/BIG/dis/$distro_version/SRPMS/$media/", @special_srpm_dir) {
+	print "Checking SRPMS dir $dir\n";
+	opendir my $rpmdir, $dir; 
+	foreach my $srpm (readdir $rpmdir) {
+		if ($srpm =~ /^\@\d+:(.*)/) {
+			link "$dir/$srpm", "$dir/$1";
+			# unlink "$dir/$srpm";
+			$srpm = $1
+		}
+		$srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
+		if ($srpm =~ /$unwanted_packages/) { next }
+		my $ok = 1;
+		if (check_version($srpm)) { 
+			defined $::failure->{$srpm} && defined $::failure->{$srpm}{$my_arch} and next;
+			if (!$::queue->{$srpm}{$my_arch} && !$::queue->{$srpm}{noarch}) {
+				my $hdr = rpm2header("$dir/$srpm");
+				check_arch($hdr) and next;
+				my $changelog = $hdr->queryformat("%{CHANGELOGNAME}");
+				my ($mail) = $changelog =~ /<(.*@.*)>/;
+				$maint{$srpm} = $mail;
+				print "Will try to compile $srpm\n";
+				push @todo, [ $dir , $srpm ]
+			}
+			foreach my $arch (@supported_arch) {
+				$ok &&= $::queue->{$srpm}{$arch} || $::queue->{$srpm}{noarch}
+			}
+		}
+		if ($clean && ($rep{$srpm} || $ok)) {
+			print "Cleaning $dir/$srpm\n";
+			unlink "$dir/build/$srpm";
+			unlink "$dir/$srpm"
+		}
+		$rep{$srpm} = 1
+	}
+	$clean = 1;
+	closedir $rpmdir
+}
+
+dump_cache();
+
+if (!@todo) {
+    print "Nothing to do\n";
+    exit
+}
+
+print "Checking basesystem tar\n";
+
+my $chroot = "$local_home/chroot";
+my $chroot_tar = "$chroot-$distro_tag.$my_arch.tar.gz";
+perform_command("sudo ~warly/files/cvs//mdk/soft/rpm-rebuilder/install-chroot-tar.sh cooker /mnt/BIG/distrib/$distro_version/$my_arch/media/main/ $chroot_tar $chroot 501 basesystem rpm-build urpmi", 'maintainers@mandriva.com', "[REBUILD] Creating the inital chroot for $distro_tag on $my_arch failed", 'chroot_inititialization',1);
+
+if (!-d "$local_home/$distro_tag/$my_arch/") {
+	mkdir "$local_home/$distro_tag";
+	if (!-d "$local_home/$distro_tag/$my_arch") {
+		mkdir "$local_home/$distro_tag/$my_arch"
+	}
+}
+
+my %done;
+foreach my $t (@todo) {
+	my ($dir, $srpm) = @$t;
+	$done{$srpm} and next;
+	$done{$srpm} = 1;
+	check_version($srpm) or next;
+	print "Installing a new chroot for $srpm in $chroot\n";
+	-d $chroot and perform_command("sudo rm -rf $chroot", 'warly@mandriva.com', "[REBUILD] Deleting of old chroot $chroot failed", 'chroot_deletion', 1);
+	mkdir $chroot;
+	perform_command("pushd $chroot && sudo tar xvf $chroot_tar", 'warly@mandriva.com', "[REBUILD] creating the initial chroot $chroot failed", 'chroot_init', 1);
+
+	dump_rpmmacros("$chroot/home/builder/.rpmmacros");
+
+	my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next;
+	my $maintainer = `rpmmon -s -p $srpm_name`;
+	my $cc = "$maint{$srpm}, maintainers\@mandriva.com";
+	chomp $maintainer;
+	if (!$maintainer) {
+		$maintainer = $cc;
+		$cc = 'maintainers@mandriva.com'
+	}
+	#($maintainer, $cc) = ('warly@mandriva.com','');
+	print "Installing build dependencies of $srpm...\n";
+	# FIXME unfortunately urpmi stalls quite often
+	system(qq{sudo pkill -9 -u root -f "urpmi --root $chroot"});
+	perform_command("sudo urpmi --root $chroot --no-verify-rpm -s --auto $dir/$srpm", $maintainer, "[REBUILD] install of build dependencies of $srpm failed on $my_arch", "build_deps_$srpm", 0, 600, $cc) or next;
+	print "Copying $srpm to $chroot\n";
+	perform_command("sudo cp $dir/$srpm $chroot/home/builder/rpm/SRPMS/", 'warly@mandriva.com', "[REBUILD] cannot copy $srpm to $chroot", "copy_$srpm", 1);
+	print "Compiling $srpm\n";
+	#system(qq{sudo chroot $chroot /bin/su builder -c "mkdir rpm/RPMS/x86_64 rpm/RPMS/noarch"});
+	if (!perform_command(qq{TMP=/home/builder/tmp/ sudo chroot $chroot /bin/su builder -c "rpm --rebuild /home/builder/rpm/SRPMS/$srpm"}, $maintainer, "[REBUILD] $srpm from $distro_tag does not build correctly on $my_arch", $srpm, 0, 18000, $cc)) {
+		$::failure->{$srpm}{$my_arch} = 1;
+		next
+	}
+	# FIXME try to install the packages afterwards
+	if (!perform_command("sudo urpmi --root $chroot --no-verify-rpm --auto $chroot/home/builder/rpm/RPMS/*/*.rpm", $maintainer, "[REBUILD] binaries packages generated from $srpm do not install correctly", "binary_test_$srpm", 0, 300)) {
+		$::failure->{$srpm}{$my_arch} = 1;
+		next
+	}
+	system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_home/$distro_tag/$my_arch/") and print "ERROR: could not copy rpm files frpm $chroot/home/builder/rpm/RPMS/ to $local_home/$distro_tag/$my_arch/ ($!)\n";
+	process_queue()
+}
+
+process_queue();
+
+dump_cache();
+
+print "ERROR: RPM with a wrong SRPM name:\n" if @wrong_rpm;
+foreach (@wrong_rpm) {
+	print "$_->[1] -> $_->[0] (",$::rpm_srpm{$_->[1]},")\n"
+}
+
+unlink $pidfile;
+
+exit;
+
+sub usage {
+	my ($error) = @_;
+	print "
+	ERROR iurt: $error" if $error;
+	print "
+	
+	usage:
+	
+		iurt [options] <distro version> <arch> <media>
+			e.g. iurt community/2006.0 x86_64 main
+
+	options:
+		--check_queue <queue base dir>
+			e.g iurt --check_queue $home/uploads/contrib/build/ cooker x86_64 contrib
+
+
+";
+	exit
+}
+
+sub process_queue {
+	my $dir = "$local_home/$distro_tag/$my_arch";
+	#my $dir = "$base_dir/RPMS/";
+	opendir my $rpmdir, $dir;
+
+	foreach my $rpm (readdir $rpmdir) {
+		my ($rarch, $srpm) = update_srpm($dir, $rpm) or next;
+		# try to keep the opportunity to prevent disk full
+		system("mv $dir/$rpm $base_dir/RPMS/") and next;
+		$::queue->{$srpm}{$rarch} = 1
+	}
+	closedir $rpmdir;
+}
+
+sub update_srpm {
+	my ($dir, $rpm) = @_;
+	my ($arch) = $rpm =~ /([^\.]+)\.rpm$/ or return 0;
+	my $srpm = $::rpm_srpm->{$rpm};
+	if (!$srpm) {
+		my $hdr = rpm2header("$dir/$rpm");
+		$hdr or return 0;
+		$srpm = $hdr->queryformat("%{SOURCERPM}");
+		$::rpm_srpm->{$rpm} = $srpm
+	}
+	$srpm = fix_srpm_name($srpm, $rpm);
+	$arch, $srpm
+}
+
+sub dump_cache {
+	my $filename = $cache;
+	open my $file, ">$filename.tmp" or die "FATAL dump_cache: cannot open $filename.tmp";
+	flock($file,LOCK_EX);
+	seek($file, 0, 2);
+	$Data::Dumper::Indent = 1;
+	print $file Data::Dumper->Dump([ $::rpm_srpm, $::failure, $::queue, $::warning ], [ "::rpm_srpm", "::failure", "::queue", "::warning" ]);
+	print $file "1";
+	unlink $filename;
+	link "$filename.tmp", $filename;
+	flock($file,LOCK_UN)
+}
+
+sub sendmail {
+	my ($to, $cc, $subject, $text, $from, $debug) = @_;
+	do { print "Cannot find sender-email-address [$to]\n"; return } unless defined($to);
+	$from ||= "Iurt the rebuild bot <warly\@mandriva.com>";
+	my $MAIL;
+	if (!$debug) { open $MAIL,  "| /usr/sbin/sendmail -t" } else { open $MAIL,  ">&STDOUT" }
+	my $sender = encode_mimewords($to);
+	my $subject = encode_mimewords($subject);
+	print $MAIL "To: $to\n";
+	if ($cc) { print $MAIL "Cc: $cc\n" }
+	print $MAIL "From: $from\n";
+	print $MAIL "Subject: $subject\n";
+	print $MAIL "\n";
+	print $MAIL $text; 
+	close($MAIL)
+}
+
+sub check_arch {
+	my ($hdr) = @_;
+	my (@exclusive_arch) = $hdr->queryformat('%{EXCLUSIVEARCH}');
+	grep { $_ eq $my_arch || $_ eq '(none)' } @exclusive_arch or return 1; 
+	my (@exclude_arch) = $hdr->queryformat('%{EXCLUDEARCH}');
+	grep { $_ eq $my_arch } @exclusive_arch and return 1 
+}
+
+sub check_version {
+	my ($srpm) = @_;
+	my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm/;
+	if (URPM::ranges_overlap("= $srpm",">= $srpm_version{$srpm_name}")) {
+		$srpm_version{$srpm_name} = $srpm;
+		return 1
+	}
+	0
+}
+
+sub fix_srpm_name {
+	my ($srpm, $rpm) = @_;
+	my $old_srpm = $srpm;
+	if ($srpm =~ s/^lib64/lib/){
+		push @wrong_rpm, [ $old_srpm, $rpm ];
+		$::rpm_srpm->{$rpm} = $srpm
+	}
+	$srpm
+}
+
+sub perform_command {
+	my ($command, $mail, $error, $hash, $die, $timeout, $cc) = @_;
+	$timeout ||= 300;
+	print "Timeout $timeout\n";
+	my $p_pid = $$;
+	my $pid = fork;
+	my $stat;
+	if ($pid) {
+		print "$command\n";
+		my $output = `$command 2>&1`;
+		# FIXME process goes in defunct always, kill -14 should unsleep it, but not
+		kill 9, $pid;
+		if ($?) {
+			if ($::warning->{$hash}{$my_arch}{$mail} % 24) {
+				sendmail($mail, $cc , $error , $output, 0, 0);
+			} else {
+				sendmail('warly@mandriva.com', '' , $error, $output, 0, 0);
+			}
+			$::warning->{$hash}{$my_arch}{$mail}++;
+			print $output;
+			die "FATAL iurt: $error." if $die;
+			return 0
+		}
+	} else {
+		my $t = sleep $timeout;
+		my ($cmd) = $command =~ s/^sudo //;
+		my $pgrep=`pgrep -f "$cmd"`;
+		print "PGREP $pgrep timeout $t\n";
+		if ($t == $timeout && $pgrep) {
+			print "\nKilling -9 $cmd\n\n";
+			system(qq{sudo pkill -9 -u root -f "$cmd"});
+		}
+		exit
+	}
+	1
+}
+
+sub kill_for_good {
+	my ($pid) = @_;
+	kill 14, $pid;
+	sleep 2;
+	if (getpgrp $pid != -1) {
+		kill 15, $pid;
+		sleep 2;
+		if (getpgrp $pid  != -1) {
+			kill 9, $pid
+		}
+	}
+}
+
+sub dump_rpmmacros {
+	my ($file) = @_;
+	open my $f, qq{| sudo sh -c "cat > $file"};
+	print $f qq{\%_topdir                \%(echo \$HOME)/rpm
+\%_tmppath               \%(echo \$HOME)/rpm/tmp/
+\%distribution           Mandriva Linux
+\%vendor                 Mandriva
+\%packager               Iurt <warly\@mandriva.com>}
+}