diff options
| -rwxr-xr-x | iurt2 | 430 |
1 files changed, 430 insertions, 0 deletions
@@ -0,0 +1,430 @@ +#!/usr/bin/perl +# +# Copyright (C) 2005 Mandrakesoft +# Copyright (C) 2005 Mandriva +# +# Author: Florent Villard <warly@mandraesoft.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# compare and rebuild packages on different architecture +# +use strict; +use Hdlist; +use Data::Dumper; +use URPM; +use MIME::Words qw(encode_mimewords); +use Fcntl ':flock'; +my $home = '/home/mandrake/'; +my $cache_home = "$home/.bugs/"; +my @supported_arch = ('i586', 'x86_64'); +my $upload = "$home/uploads/"; +my $local_home = '/export/home/mandrake'; +my $base_dir = $upload; +my $check_queue = 0; +if (@ARGV[0] eq '--check_queue') { + $check_queue = 1; + shift @ARGV; + $base_dir = shift @ARGV; +} + +my $distro_version = shift @ARGV; +my $distro_tag = $distro_version; +$distro_tag =~ s,/,-,g; +my $cache = "$cache_home/iurt.$distro_tag.cache"; +print "Loading cache file $cache\n"; +eval { require $cache }; +$::rpm_srpm ||= {}; +$::failure ||= {}; +$::queue ||= {}; +$::warning ||= {}; +my $my_arch = shift @ARGV; +my $media = shift @ARGV; +my @special_srpm_dir = @ARGV; + +my $unwanted_packages = '^monotone-'; + +$my_arch or usage(); +if ($check_queue) { + -d $base_dir or usage("$base_dir does not exist"); +} else { + $base_dir .= $distro_version; + $base_dir =~ s/community//g; + if ($distro_version ne 'cooker') { + if ($media ne 'main') { + $base_dir .= "/$media" + } + } elsif ($media eq 'contrib') { + $base_dir = "$upload/contrib" + } + -d $base_dir or usage("$base_dir does not exist") +} + +my $pidfile = "$cache_home/iurt.$distro_tag.$my_arch.pid"; +my (@stat) = stat $pidfile; +-f $pidfile and do { + open PID, $pidfile; + my $pid = <PID>; + close PID; + if ($pid && getpgrp $pid != -1) { + if ($stat[9] < time - 36000) { + print "An other iurt pid $pid is running for a very long time, killing it\n"; + my $i; + while ($i < 5 && getpgrp $pid != -1) { + kill 9, $pid; + $i++; + sleep 1 + } + } else { + print "An other iurt is running for $my_arch, pid $pid\n"; + exit + } + } else { + print "A previous iurt for $my_arch seems dead, cleaning.\n"; + unlink $pidfile + } +}; + +open PID, ">$pidfile"; +print PID $$; +close PID; + +my %big; +my %srpm_version; +my @wrong_rpm; +foreach my $arch (@supported_arch) { + my $rpms_dir = "/mnt/BIG/dis/$distro_version/$arch/media/$media/"; + print "Checking current packages in $rpms_dir\n"; + opendir my $rpmdir, $rpms_dir; + foreach my $rpm (readdir $rpmdir) { + my ($rarch, $srpm) = update_srpm($rpms_dir, $rpm) or next; + $big{$rpm} = 1; + $::queue->{$srpm}{$arch} = 1; + $::queue->{$srpm}{$rarch} = 1; + check_version($srpm) + } + closedir $rpmdir +} + +if ($check_queue) { + my $dir = "$base_dir/build/RPMS/"; + opendir my $rpmdir, $dir; + + print "Checking upload queue in $dir\n"; + foreach my $rpm (readdir $rpmdir) { + $rpm =~ /\.rpm$/ or next; + if ($big{$rpm}) { + print "Cleaning $rpm\n"; + unlink "$dir/$rpm"; + next + } + my ($arch, $srpm) = update_srpm($dir, $rpm) or next; + $::queue->{$srpm}{$arch} = 1; + check_version($srpm) + } + closedir $rpmdir; + +} + +my %maint; + +my @todo; +-d "$base_dir/build/$my_arch" or mkdir "$base_dir/build/$my_arch"; +push @special_srpm_dir, "$base_dir/build/SRPMS/" if $check_queue; +my $clean; +my %rep; +my %done_rpm; +# +# FIXME all the rep but the first one are cleaned +# +foreach my $dir ("/mnt/BIG/dis/$distro_version/SRPMS/$media/", @special_srpm_dir) { + print "Checking SRPMS dir $dir\n"; + opendir my $rpmdir, $dir; + foreach my $srpm (readdir $rpmdir) { + if ($srpm =~ /^\@\d+:(.*)/) { + link "$dir/$srpm", "$dir/$1"; + # unlink "$dir/$srpm"; + $srpm = $1 + } + $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; + if ($srpm =~ /$unwanted_packages/) { next } + my $ok = 1; + if (check_version($srpm)) { + defined $::failure->{$srpm} && defined $::failure->{$srpm}{$my_arch} and next; + if (!$::queue->{$srpm}{$my_arch} && !$::queue->{$srpm}{noarch}) { + my $hdr = rpm2header("$dir/$srpm"); + check_arch($hdr) and next; + my $changelog = $hdr->queryformat("%{CHANGELOGNAME}"); + my ($mail) = $changelog =~ /<(.*@.*)>/; + $maint{$srpm} = $mail; + print "Will try to compile $srpm\n"; + push @todo, [ $dir , $srpm ] + } + foreach my $arch (@supported_arch) { + $ok &&= $::queue->{$srpm}{$arch} || $::queue->{$srpm}{noarch} + } + } + if ($clean && ($rep{$srpm} || $ok)) { + print "Cleaning $dir/$srpm\n"; + unlink "$dir/build/$srpm"; + unlink "$dir/$srpm" + } + $rep{$srpm} = 1 + } + $clean = 1; + closedir $rpmdir +} + +dump_cache(); + +if (!@todo) { + print "Nothing to do\n"; + exit +} + +print "Checking basesystem tar\n"; + +my $chroot = "$local_home/chroot"; +my $chroot_tar = "$chroot-$distro_tag.$my_arch.tar.gz"; +perform_command("sudo ~warly/files/cvs//mdk/soft/rpm-rebuilder/install-chroot-tar.sh cooker /mnt/BIG/distrib/$distro_version/$my_arch/media/main/ $chroot_tar $chroot 501 basesystem rpm-build urpmi", 'maintainers@mandriva.com', "[REBUILD] Creating the inital chroot for $distro_tag on $my_arch failed", 'chroot_inititialization',1); + +if (!-d "$local_home/$distro_tag/$my_arch/") { + mkdir "$local_home/$distro_tag"; + if (!-d "$local_home/$distro_tag/$my_arch") { + mkdir "$local_home/$distro_tag/$my_arch" + } +} + +my %done; +foreach my $t (@todo) { + my ($dir, $srpm) = @$t; + $done{$srpm} and next; + $done{$srpm} = 1; + check_version($srpm) or next; + print "Installing a new chroot for $srpm in $chroot\n"; + -d $chroot and perform_command("sudo rm -rf $chroot", 'warly@mandriva.com', "[REBUILD] Deleting of old chroot $chroot failed", 'chroot_deletion', 1); + mkdir $chroot; + perform_command("pushd $chroot && sudo tar xvf $chroot_tar", 'warly@mandriva.com', "[REBUILD] creating the initial chroot $chroot failed", 'chroot_init', 1); + + dump_rpmmacros("$chroot/home/builder/.rpmmacros"); + + my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/ or next; + my $maintainer = `rpmmon -s -p $srpm_name`; + my $cc = "$maint{$srpm}, maintainers\@mandriva.com"; + chomp $maintainer; + if (!$maintainer) { + $maintainer = $cc; + $cc = 'maintainers@mandriva.com' + } + #($maintainer, $cc) = ('warly@mandriva.com',''); + print "Installing build dependencies of $srpm...\n"; + # FIXME unfortunately urpmi stalls quite often + system(qq{sudo pkill -9 -u root -f "urpmi --root $chroot"}); + perform_command("sudo urpmi --root $chroot --no-verify-rpm -s --auto $dir/$srpm", $maintainer, "[REBUILD] install of build dependencies of $srpm failed on $my_arch", "build_deps_$srpm", 0, 600, $cc) or next; + print "Copying $srpm to $chroot\n"; + perform_command("sudo cp $dir/$srpm $chroot/home/builder/rpm/SRPMS/", 'warly@mandriva.com', "[REBUILD] cannot copy $srpm to $chroot", "copy_$srpm", 1); + print "Compiling $srpm\n"; + #system(qq{sudo chroot $chroot /bin/su builder -c "mkdir rpm/RPMS/x86_64 rpm/RPMS/noarch"}); + if (!perform_command(qq{TMP=/home/builder/tmp/ sudo chroot $chroot /bin/su builder -c "rpm --rebuild /home/builder/rpm/SRPMS/$srpm"}, $maintainer, "[REBUILD] $srpm from $distro_tag does not build correctly on $my_arch", $srpm, 0, 18000, $cc)) { + $::failure->{$srpm}{$my_arch} = 1; + next + } + # FIXME try to install the packages afterwards + if (!perform_command("sudo urpmi --root $chroot --no-verify-rpm --auto $chroot/home/builder/rpm/RPMS/*/*.rpm", $maintainer, "[REBUILD] binaries packages generated from $srpm do not install correctly", "binary_test_$srpm", 0, 300)) { + $::failure->{$srpm}{$my_arch} = 1; + next + } + system("cp $chroot/home/builder/rpm/RPMS/*/*.rpm $local_home/$distro_tag/$my_arch/") and print "ERROR: could not copy rpm files frpm $chroot/home/builder/rpm/RPMS/ to $local_home/$distro_tag/$my_arch/ ($!)\n"; + process_queue() +} + +process_queue(); + +dump_cache(); + +print "ERROR: RPM with a wrong SRPM name:\n" if @wrong_rpm; +foreach (@wrong_rpm) { + print "$_->[1] -> $_->[0] (",$::rpm_srpm{$_->[1]},")\n" +} + +unlink $pidfile; + +exit; + +sub usage { + my ($error) = @_; + print " + ERROR iurt: $error" if $error; + print " + + usage: + + iurt [options] <distro version> <arch> <media> + e.g. iurt community/2006.0 x86_64 main + + options: + --check_queue <queue base dir> + e.g iurt --check_queue $home/uploads/contrib/build/ cooker x86_64 contrib + + +"; + exit +} + +sub process_queue { + my $dir = "$local_home/$distro_tag/$my_arch"; + #my $dir = "$base_dir/RPMS/"; + opendir my $rpmdir, $dir; + + foreach my $rpm (readdir $rpmdir) { + my ($rarch, $srpm) = update_srpm($dir, $rpm) or next; + # try to keep the opportunity to prevent disk full + system("mv $dir/$rpm $base_dir/RPMS/") and next; + $::queue->{$srpm}{$rarch} = 1 + } + closedir $rpmdir; +} + +sub update_srpm { + my ($dir, $rpm) = @_; + my ($arch) = $rpm =~ /([^\.]+)\.rpm$/ or return 0; + my $srpm = $::rpm_srpm->{$rpm}; + if (!$srpm) { + my $hdr = rpm2header("$dir/$rpm"); + $hdr or return 0; + $srpm = $hdr->queryformat("%{SOURCERPM}"); + $::rpm_srpm->{$rpm} = $srpm + } + $srpm = fix_srpm_name($srpm, $rpm); + $arch, $srpm +} + +sub dump_cache { + my $filename = $cache; + open my $file, ">$filename.tmp" or die "FATAL dump_cache: cannot open $filename.tmp"; + flock($file,LOCK_EX); + seek($file, 0, 2); + $Data::Dumper::Indent = 1; + print $file Data::Dumper->Dump([ $::rpm_srpm, $::failure, $::queue, $::warning ], [ "::rpm_srpm", "::failure", "::queue", "::warning" ]); + print $file "1"; + unlink $filename; + link "$filename.tmp", $filename; + flock($file,LOCK_UN) +} + +sub sendmail { + my ($to, $cc, $subject, $text, $from, $debug) = @_; + do { print "Cannot find sender-email-address [$to]\n"; return } unless defined($to); + $from ||= "Iurt the rebuild bot <warly\@mandriva.com>"; + my $MAIL; + if (!$debug) { open $MAIL, "| /usr/sbin/sendmail -t" } else { open $MAIL, ">&STDOUT" } + my $sender = encode_mimewords($to); + my $subject = encode_mimewords($subject); + print $MAIL "To: $to\n"; + if ($cc) { print $MAIL "Cc: $cc\n" } + print $MAIL "From: $from\n"; + print $MAIL "Subject: $subject\n"; + print $MAIL "\n"; + print $MAIL $text; + close($MAIL) +} + +sub check_arch { + my ($hdr) = @_; + my (@exclusive_arch) = $hdr->queryformat('%{EXCLUSIVEARCH}'); + grep { $_ eq $my_arch || $_ eq '(none)' } @exclusive_arch or return 1; + my (@exclude_arch) = $hdr->queryformat('%{EXCLUDEARCH}'); + grep { $_ eq $my_arch } @exclusive_arch and return 1 +} + +sub check_version { + my ($srpm) = @_; + my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm/; + if (URPM::ranges_overlap("= $srpm",">= $srpm_version{$srpm_name}")) { + $srpm_version{$srpm_name} = $srpm; + return 1 + } + 0 +} + +sub fix_srpm_name { + my ($srpm, $rpm) = @_; + my $old_srpm = $srpm; + if ($srpm =~ s/^lib64/lib/){ + push @wrong_rpm, [ $old_srpm, $rpm ]; + $::rpm_srpm->{$rpm} = $srpm + } + $srpm +} + +sub perform_command { + my ($command, $mail, $error, $hash, $die, $timeout, $cc) = @_; + $timeout ||= 300; + print "Timeout $timeout\n"; + my $p_pid = $$; + my $pid = fork; + my $stat; + if ($pid) { + print "$command\n"; + my $output = `$command 2>&1`; + # FIXME process goes in defunct always, kill -14 should unsleep it, but not + kill 9, $pid; + if ($?) { + if ($::warning->{$hash}{$my_arch}{$mail} % 24) { + sendmail($mail, $cc , $error , $output, 0, 0); + } else { + sendmail('warly@mandriva.com', '' , $error, $output, 0, 0); + } + $::warning->{$hash}{$my_arch}{$mail}++; + print $output; + die "FATAL iurt: $error." if $die; + return 0 + } + } else { + my $t = sleep $timeout; + my ($cmd) = $command =~ s/^sudo //; + my $pgrep=`pgrep -f "$cmd"`; + print "PGREP $pgrep timeout $t\n"; + if ($t == $timeout && $pgrep) { + print "\nKilling -9 $cmd\n\n"; + system(qq{sudo pkill -9 -u root -f "$cmd"}); + } + exit + } + 1 +} + +sub kill_for_good { + my ($pid) = @_; + kill 14, $pid; + sleep 2; + if (getpgrp $pid != -1) { + kill 15, $pid; + sleep 2; + if (getpgrp $pid != -1) { + kill 9, $pid + } + } +} + +sub dump_rpmmacros { + my ($file) = @_; + open my $f, qq{| sudo sh -c "cat > $file"}; + print $f qq{\%_topdir \%(echo \$HOME)/rpm +\%_tmppath \%(echo \$HOME)/rpm/tmp/ +\%distribution Mandriva Linux +\%vendor Mandriva +\%packager Iurt <warly\@mandriva.com>} +} |