aboutsummaryrefslogtreecommitdiffstats
path: root/iurt2
diff options
context:
space:
mode:
Diffstat (limited to 'iurt2')
-rwxr-xr-xiurt2639
1 files changed, 429 insertions, 210 deletions
diff --git a/iurt2 b/iurt2
index 61bef29..0b6c52b 100755
--- a/iurt2
+++ b/iurt2
@@ -50,7 +50,7 @@ $run{todo} = [ ];
#
[ "", "$program_name", 0, "[--cache] [--chrooted-urpmi <media prefix>] [--concurrent-run] [--config foo value] [--warn] [--verbose integer]
[--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}]
- [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--unionfs]
+ [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] [--unionfs]
[--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--unionfs]
{--config_help |
--chroot --arch {i586|x86_64|ppc} --distro {cooker|2006.0|community/2006.0|...} } |
@@ -102,6 +102,9 @@ $run{todo} = [ ];
[ "", "debug", 0, "",
"Activate debug mode",
sub { $run{debug} = 1 }, "Activating debug mode" ],
+ [ "g", "group", 0, "",
+ "Activate group mode, packages will be compiled as a global set, not as individual packages",
+ sub { $run{group} = 1 }, "Activating group mode to compile packages as a set" ],
[ "u", "unionfs", 0, "",
"Activate unionfs mode",
sub { $run{unionfs} = 1 }, "Activating unionfs mode" ],
@@ -134,7 +137,7 @@ $run{todo} = [ ];
my $hdr = rpm2header($_);
if (check_arch($hdr)) {
print {$run{LOG}} "iurt: force build for $2 (from $1)\n";
- push @{$run{todo}}, [ $path, $srpm ]
+ push @{$run{todo}}, [ $path, $srpm, 1 ]
} else {
print {$run{LOG}} "ERROR iurt: $_ could not be build on $run{my_arch}, ignored.\n"
}
@@ -221,10 +224,21 @@ my $urpmi_options = "-v --no-verify-rpm --nolock --auto --ignoresize";
if ($run{use_system_distrib}) {
$config->{basesystem_media_root} ||= $run{use_system_distrib}
} elsif ($run{chrooted_urpmi}) {
- $run{chrooted_media} = "$run{chrooted_urpmi}/$run{distro}/$run{my_arch}"
+ my ($host) = $run{chrooted_urpmi} =~ m,(?:file|http|ftp)://([^/]*),;
+ my ($name,$aliases,$addrtype,$length,@addrs) = gethostbyname $host;
+ my $ip = join '.', unpack('C4',$addrs[0]);
+ $ip =~ /\d+\.\d+\.\d+\.\d+/ or die "FATAL iurt: could not resolve $host ip address";
+ $run{chrooted_urpmi} =~ s/$host/$ip/;
+ $run{chrooted_media} = "$run{chrooted_urpmi}/$run{distro}/$run{my_arch}";
+ print {$run{LOG}} "iurt: using $run{chrooted_media} as insallation media\n" if $run{verbose} > 2
} else {
$urpmi_options .= " --use-distrib $config->{repository}/$run{distro}/$run{my_arch}"
}
+
+if (!$run{chrooted_urpmi} && $run{group}) {
+ die "FATAL iurt: option --chrooted-urpmi is mandatory if --group is selected"
+}
+
my %config_usage = (
admin => { desc => 'Mail of the administrator of packages builds', default => '' },
all_media => { desc => 'List of known media', default => [ 'main', 'contrib' ] },
@@ -291,7 +305,7 @@ my $chroot_name = "chroot_$run{distro_tag}$debug_tag";
my $chroot = "$config->{local_home}/$chroot_name";
my $chroot_tar = "$chroot.$run{my_arch}.tar.gz";
if ($run{chroot}) {
- check_chroot($chroot, $chroot_tar, \%run)
+ check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot"
}
my $cache;
@@ -395,8 +409,8 @@ if ($df->{per} == 100) {
}
if ($run{shell}) {
- my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id);
- add_local_user($chroot_tmp, \%run, $luser, $run{uid});
+ ($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or die "FATAL iurt: could not create temporary chroot";
+ add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL iurt: could not add local user";
$run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp";
add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi');
add_sudoers(\%run, $luser);
@@ -430,7 +444,7 @@ if ($config->{rsync_to} && !$run{no_rsync}) {
print {$run{LOG}} "iurt: try to dump rpm macros to $chroot/home/builder/.rpmmacros\n" if $run{verbose} > 3;
if (!dump_rpmmacros("$chroot/home/builder/.rpmmacros")) {
print "ERROR iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros, trying to rebuild the chroot";
- check_chroot($chroot, $chroot_tar, \%run);
+ check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot";
dump_rpmmacros("$chroot/home/builder/.rpmmacros") or die "FATAL iurt: could not dump rpm macros to $chroot/home/builder/.rpmmacros"
}
@@ -440,7 +454,7 @@ my $s = sub {
dump_cache(\%run);
$Data::Dumper::Indent = 0;
$Data::Dumper::Terse = 1;
- print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n";
+ print {$run{LOG}} "Running environment:\n", Data::Dumper->Dump([\%run]), "\n\n" if $run{verbose} > 5;
print {$run{LOG}} "Configuration:\n",Data::Dumper->Dump([$config]),"\n\n";
}
exit
@@ -449,8 +463,18 @@ $SIG{TERM} = $s;
$SIG{INT} = $s;
$run{main} = 1;
-foreach (my $i ; $i < @{$run{todo}}; $i++) {
- my ($dir, $srpm) = @{$run{todo}[$i]};
+my $local_media;
+my $rebuild;
+if ($run{group}) {
+ $rebuild = 1;
+ order_packages(\%run) or die "FATAL iurt: could not order packages";
+ $local_media = "$local_spool/$run{run}/"
+}
+
+do {
+ foreach (my $i ; $i < @{$run{todo}}; $i++) {
+ my ($dir, $srpm, $status) = @{$run{todo}[$i]};
+ $status or next;
$done{$srpm} and next;
$done{$srpm} = 1;
check_version($srpm) or next;
@@ -460,20 +484,37 @@ foreach (my $i ; $i < @{$run{todo}}; $i++) {
print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n";
# FIXME unfortunately urpmi stalls quite often
my $retry = 0;
-retry:
- if (!$run{chrooted_urpmi}) {
+ retry:
+ if (!$run{chrooted_urpmi}) {
my $match = "urpmi $urpmi_options --root $chroot_tmp";
if (!clean_process($match, $run{verbose})) {
- dump_cache(\%run);
- die "FATAL iurt: Could not have urpmi working !"
+ dump_cache(\%run);
+ die "FATAL iurt: Could not have urpmi working !"
}
}
- my ($unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm);
+ ($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id, $srpm) or next;
if ($run{chrooted_urpmi}) {
$run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp ";
add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi');
add_media(\%run, $config, $chroot_tmp);
+ if (-d "$local_media/hdlist.cz") {
+ mkdir("$chroot_tmp/iurt_media/");
+ opendir my $dir, $local_media;
+ my $next;
+ foreach my $f (readdir $dir) {
+ $f =~ /(\.rpm|^hdlist.cz)$/ or next;
+ if (!link "$local_media/$f", "$chroot_tmp/iurt_media") {
+ if (!copy "$local_media/$f", "$chroot_tmp/iurt_media") {
+ print {$run{LOG}} "ERROR iurt: could not copy file $local_media/$f to $chroot_tmp/iurt_media";
+ $next = 1;
+ last
+ }
+ }
+ }
+ next if $next;
+ add_local_media(\%run, $config, $chroot_tmp, "/iurt_media")
+ }
$run{urpmi_command} = "chroot $chroot_tmp urpmi $urpmi_options "
} else {
$run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp"
@@ -491,71 +532,36 @@ retry:
}
#($maintainer, $cc) = ($config->{admin},'');
- print {$run{LOG}} "iurt: adding local user to the chroot...\n" if $run{verbose};
- add_local_user($chroot_tmp, \%run, $luser, $run{uid});
+ print {$run{LOG}} "iurt: adding local user $luser into $chroot_tmp...\n" if $run{verbose};
+ add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or next;
- # recreate a new srpm for buildarch condition in the spec file
- print {$run{LOG}} "iurt: copying $srpm to $chroot_tmp\n" if $run{verbose} > 1;
- perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/",
- \%run, $config,
- mail => $config->{admin},
- error => "[REBUILD] cannot copy $srpm to $chroot_tmp",
- debug_mail => $run{debug},
- hash => "copy_$srpm") or next;
-
- my %opt = ( mail => $config->{admin},
- error => "[REBUILD] cannot install $srpm in $chroot_tmp",
- debug_mail => $run{debug},
- hash => "install_$srpm",
- retry => $retry,
- callback => sub {
- my ($opt, $output) = @_;
- print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
- if ($output =~ /user $luser does not exist/) {
- print {$run{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run{verbose} > 1;
- $opt->{error} = "[CHROOT] chroot is corrupted";
- $opt->{retry} = 1 if !$opt->{retry};
- return
- }
- 1
- }
- );
- print {$run{LOG}} "iurt: recreating src.rpm...\n" if $run{verbose};
- if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"},
- \%run, $config, %opt)) {
- print {$run{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run{debug};
- if ($opt{retry}) {
- $retry = 1;
- check_chroot($chroot, $chroot_tar, \%run);
- goto retry
- }
- die;
- next
- }
- # This should not be necessairy any more if urpmi *.spec works, but it doesn't
- #
- perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm --nodeps -bs /home/$luser/rpm/SPECS/*.spec"},
- \%run, $config,
- mail => $config->{admin},
- error => "[REBUILD] cannot create $srpm in $chroot_tmp",
- debug_mail => $run{debug},
- hash => "create_$srpm") or next;
- print {$run{LOG}} "iurt: installing build dependencies of $srpm...\n" if $run{verbose};
+ my $ret = recreate_srpm(\%run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry);
+ if ($ret == -1) {
+ $retry = 1;
+ goto retry
+ } elsif (!$ret) {
+ next
+ }
+ print {$run{LOG}} "iurt: installing build dependencies of $srpm...\n" if $run{verbose};
my $wait_urpmi = sub {
- print {$run{LOG}} "WARNING iurt: urpmi database locked, waiting...\n" if $run{debug};
- sleep 30;
- $wait_limit++;
- if ($wait_limit > 8) {
- $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null})
- } };
+ print {$run{LOG}} "WARNING iurt: urpmi database locked, waiting...\n" if $run{debug};
+ sleep 30;
+ $wait_limit++;
+ if ($wait_limit > 8) {
+ $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null})
+ } };
my $path_srpm = $run{chrooted_urpmi} ? "/home/$luser/rpm/SRPMS/" : "$chroot_tmp/home/$luser/rpm/SRPMS/";
- if (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm",
+ #
+ # on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore
+ # installed packages, retrying several time to be sure something is really broken
+ my $install_deps_retry = 5;
+ while (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm",
\%run, $config,
mail => $config->{admin},
error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}",
hash => "install_deps_$srpm",
- timeout => 600,
+ timeout => 900,
srpm => $srpm,
freq => 1,
#cc => $cc,
@@ -565,44 +571,49 @@ retry:
wait_regexp => 'database locked',
wait_callback => $wait_urpmi,
log => "$local_spool/log/$srpm/",
+ reopen_log => 1,
callback => sub {
- my ($opt, $output) = @_;
- print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
- my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
- # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
- @missing_deps or return 1;
- while (my $missing_deps = shift @missing_deps) {
- my $version = shift @missing_deps;
- my $version2 = shift @missing_deps;
- $version ||= $version2 || 0;
- my $p = $pack_provide{$missing_deps} || $missing_deps;
- my $other_maint = `rpmmon -p "$p"`;
- chomp $other_maint;
- print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
- if ($other_maint && $other_maint ne 'NOT_FOUND') {
- $opt->{mail} = "$maintainer, $other_maint";
- $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
- }
- # remember what is needed, and do not try to recompile until it is available
- push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
+ my ($opt, $output) = @_;
+ print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug};
+# 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error).
+# my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g;
+ my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g;
+ # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway
+ @missing_deps or return 1;
+ while (my $missing_deps = shift @missing_deps) {
+ my $version = shift @missing_deps;
+ my $version2 = shift @missing_deps;
+ $version ||= $version2 || 0;
+ my $p = $pack_provide{$missing_deps} || $missing_deps;
+ my $other_maint = `rpmmon -p "$p"`;
+ chomp $other_maint;
+ print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n";
+ if ($other_maint && $other_maint ne 'NOT_FOUND') {
+ $opt->{mail} = "$maintainer, $other_maint";
+ $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}";
}
- 1
+ # remember what is needed, and do not try to recompile until it is available
+ push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ];
+ }
+ 0
},
- )) {
- $run{status}{$srpm} = 'install_deps_failure';
- next
+ )) {
+ $install_deps_retry--;
+ foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
+ if (!$install_deps_retry || $cache->{needed}{$srpm}) {
+ $run{status}{$srpm} = 'install_deps_failure';
+ next
+ }
}
# try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory
# running rpm -qa several time seems to fix the problem
- foreach (1 .. 3) {
- system("sudo chroot $chroot_tmp rpm -qa &> /dev/null")
- }
+ foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") }
perform_command("sudo chroot $chroot_tmp rpm -qa",
- \%run, $config,
- hash => "rpm_qa_$srpm",
- timeout => 60,
- debug_mail => $run{debug},
- log => "$local_spool/log/$srpm/"); # or next; As this failed quite often, do not stop
+ \%run, $config,
+ hash => "rpm_qa_$srpm",
+ timeout => 60,
+ debug_mail => $run{debug},
+ log => "$local_spool/log/$srpm/"); # or next; As this failed quite often, do not stop
print {$run{LOG}} "Compiling $srpm\n" if $run{verbose};
my $command = "rpm --rebuild /home/$luser/rpm/SRPMS/$srpm";
if ($run{stop}) {
@@ -611,84 +622,85 @@ retry:
$command = "rpm -b$run{stop} /home/$luser/rpm/SPECS/*.spec"
}
if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"},
- \%run, $config,
- mail => $maintainer,
- error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}",
- hash => "build_$srpm",
- timeout => 18000,
- srpm => $srpm,
- debug_mail => $run{debug},
- cc => $cc,
- log => "$local_spool/log/$srpm/",
- error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
- callback => sub {
- my ($opt, $output) = @_;
- if ($run{stop}) {
- print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp (pid $$)\n";
- # exec does not work because it seems stdin and out are shared between children
- system("sudo chroot $chroot_tmp /bin/su $luser -c bash");
- exit
- }
- print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug};
- if ($unionfs_tmp && $output =~ /no space left on device/i) {
- print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
- $cache->{no_unionfs}{$srpm} = 1;
- return 1
- } elsif ($unionfs_tmp && $output =~ m,$home,) {
- print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n";
- $cache->{no_unionfs}{$srpm} = 1;
- return 1
- }
- 1
- },
- freq => 1)) {
- # FIXME
- # The simple algo used here is :
- # try to compile it with unionfs, if it runs out of space, compile it without the next time
- #
- # This could be improved in keeping this srpm name for future version, but if we compile it
- # on a new machine with more ram, or if next version compiles just fine with unionfs, we will
- # loose the unionfs advantage.
- #
- # Maybe the right thing to do would be to first try to increase the tmpfs size (more than 50 % of the
- # physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs
- # plus). Or to keep the faulty package a unionfs exception for some time, to save some more extra builds.
- #
- if (!$unionfs_tmp) {
- $run{status}{$srpm} = 'build_failure';
- $cache->{failure}{$srpm} = 1
- } elsif (!$cache->{no_unionfs}{$srpm}) {
- $run{status}{$srpm} = 'build_failure';
- $cache->{failure}{$srpm} = 1
- } else {
- goto retry
+ \%run, $config,
+ mail => $maintainer,
+ error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}",
+ hash => "build_$srpm",
+ timeout => 18000,
+ srpm => $srpm,
+ debug_mail => $run{debug},
+ cc => $cc,
+ log => "$local_spool/log/$srpm/",
+ error_regexp => 'rror.*ailed|Bad exit status|RPM build error',
+ callback => sub {
+ my ($opt, $output) = @_;
+ if ($run{stop}) {
+ print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp (pid $$)\n";
+ # exec does not work because it seems stdin and out are shared between children
+ system("sudo chroot $chroot_tmp /bin/su $luser -c bash");
+ exit
}
- if (!glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") {
- next
+ print {$run{LOG}} "iurt: calling callback for $opt->{hash}\n" if $run{debug};
+ if ($unionfs_tmp && $output =~ /no space left on device/i) {
+ print {$run{LOG}} "ERROR iurt: running out of space to compile $srpm in unionfs mode, will recompile it in normal mode\n";
+ $cache->{no_unionfs}{$srpm} = 1;
+ return 1
+ } elsif ($unionfs_tmp && $output =~ m,$home,) {
+ print {$run{LOG}} "ERROR iurt: seems like building $srpm needs to access /proc/self/exe, which is broken with unionfs, will try to recompile it in non unionfs mode\n";
+ $cache->{no_unionfs}{$srpm} = 1;
+ return 1
}
+ 1
+ },
+ freq => 1)) {
+ # FIXME
+ # The simple algo used here is :
+ # try to compile it with unionfs, if it runs out of space, compile it without the next time
+ #
+ # This could be improved in keeping this srpm name for future version, but if we compile it
+ # on a new machine with more ram, or if next version compiles just fine with unionfs, we will
+ # loose the unionfs advantage.
+ #
+ # Maybe the right thing to do would be to first try to increase the tmpfs size (more than 50 % of the
+ # physical RAM), but this will lead to more swap usage, and slower compilation (and lost of the unionfs
+ # plus). Or to keep the faulty package a unionfs exception for some time, to save some more extra builds.
+ #
+ if (!glob "$chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm") {
+ if ($unionfs_tmp && $cache->{no_unionfs}{$srpm}) {
+ goto retry
+ }
+ $cache->{failure}{$srpm} = 1;
+ $run{status}{$srpm} = 'build_failure';
+ next
+ }
}
# do some cleaning if the compilation is successful
delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm};
if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm",
- \%run, $config,
- mail => $maintainer,
- error => "[REBUILD] binaries packages generated from $srpm do not install correctly",
- hash => "binary_test_$srpm",
- srpm => $srpm,
- timeout => 600,
- debug_mail => $run{debug},
- freq => 1,
- wait_regexp => 'database locked',
- wait_callback => $wait_urpmi,
-error_regexp => 'unable to access',
- log => "$local_spool/log/$srpm/")) {
- $cache->{failure}{$srpm} = 1;
- $run{status}{$srpm} = 'binary_test_failure';
- next
+ \%run, $config,
+ mail => $maintainer,
+ error => "[REBUILD] binaries packages generated from $srpm do not install correctly",
+ hash => "binary_test_$srpm",
+ srpm => $srpm,
+ timeout => 600,
+ debug_mail => $run{debug},
+ freq => 1,
+ wait_regexp => 'database locked',
+ wait_callback => $wait_urpmi,
+ error_regexp => 'unable to access',
+ log => "$local_spool/log/$srpm/")) {
+ $cache->{failure}{$srpm} = 1;
+ $run{status}{$srpm} = 'binary_test_failure';
+ next
}
if ($run{debug}) {
print {$run{LOG}} "iurt: debug mode, skip other packages\n";
exit
+ } elsif ($run{group}) {
+ print {$run{LOG}} "iurt: group mode, keep packages for local media\n";
+ $run{done}{$srpm} = $done;
+ system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_media ($!)\n";
+ system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_media ($!)\n";
} else {
print {$run{LOG}} "iurt: build successful, copying packages to $local_spool.\n";
$run{status}{$srpm} = 'ok';
@@ -699,8 +711,12 @@ error_regexp => 'unable to access',
process_queue($config, \%run, \@wrong_rpm, 1)
}
# dymp_cache each time so that concurrent process can get updated
- dump_cache(\%run) if $run{concurrent_run}
-}
+ dump_cache(\%run) if $run{concurrent_run};
+ }
+ if ($run{group}) {
+ $rebuild = 1 if order_packages(\%run)
+ }
+} while ($rebuild);
print {$run{LOG}} "iurt: reprocess generated packages queue\n" if $run{verbose};
process_queue($config, \%run, \@wrong_rpm);
@@ -769,7 +785,7 @@ sub clean_all_unionfs {
}
foreach (readdir $dir) {
/unionfs\.((?:0\.)?\d+)\.(\d+)$/ or next;
- clean_unionfs($unionfs_dir, $1, $2);
+ clean_unionfs($unionfs_dir, \%run, $1, $2);
}
closedir $dir
}
@@ -777,7 +793,11 @@ sub clean_all_unionfs {
sub clean_all_chroot_tmp {
my ($chroot_dir, $prefix) = @_;
print {$run{LOG}} "Cleaning old chroot remaining dir in $chroot_dir\n" if $run{verbose};
- opendir my $dir, $chroot_dir or die "FATAL iurt: could not open $chroot_dir ($!)";
+ my $dir;
+ if (!opendir $dir, $chroot_dir) {
+ print "ERROR iurt: could not open $chroot_dir ($!)";
+ return
+ }
foreach (readdir $dir) {
/$prefix.*$/ or next;
clean_chroot_tmp($chroot_dir, $_);
@@ -837,31 +857,41 @@ sub clean {
}
sub clean_unionfs {
- my ($unionfs_dir, $run, $union_id) = @_;
- if (system("sudo umount $unionfs_dir/unionfs.$run.$union_id/proc &>/dev/null") && $run{verbose} > 1) {
- print {$run{LOG}} "ERROR iurt: could not umount /proc in $unionfs_dir/unionfs.$run.$union_id\n"
- }
- print {$run{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run{verbose} > 1;
+ my ($unionfs_dir, $run, $r, $union_id) = @_;
+ -d "$unionfs_dir/unionfs.$r.$union_id" or return $union_id;
+ print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run->{verbose} > 1;
my $nok = 1;
+ my $proc = "$unionfs_dir/unionfs.$r.$union_id/proc";
while ($nok) {
+ $nok = 0;
+ if (-d $proc && check_mounted($proc, 'proc')) {
+ print {$run->{LOG}} "iurt iurt: umounting $proc\n" if $run->{verbose} > 2;
+ if (system("sudo umount $proc &>/dev/null")) {
+ print {$run->{LOG}} "ERROR iurt: could not umount $proc\n";
+ return $union_id + 1
+ }
+ }
foreach my $t ("unionfs",'tmpfs') {
# unfortunately quite oftem the unionfs is busy and could not be unmounted
- my $d = "$unionfs_dir/$t.$run.$union_id";
+ my $d = "$unionfs_dir/$t.$r.$union_id";
my $last;
- while (check_mounted($d, $t)) {
+ if (-d $d && check_mounted($d, $t)) {
+ $nok = 1;
system("sudo /sbin/fuser -k $d &> /dev/null");
- print {$run{LOG}} "iurt: umounting $d\n" if $run{verbose} > 2;
+ print {$run->{LOG}} "iurt: umounting $d\n" if $run->{verbose} > 2;
if (system(qq{sudo umount $d &> /dev/null})) {
- print {$run{LOG}} "WARNING iurt: could not umount $d ($!)\n" if $run{verbose} > 1;
- $union_id++;
- $last = 1;
- last
+ print {$run->{LOG}} "WARNING iurt: could not umount $d ($!)\n" if $run->{verbose} > 1;
+ return $union_id + 1;
}
}
- $last and last;
- print {$run{LOG}} "iurt: removing $d\n" if $run{verbose} > 1;
- system(qq{sudo rm -rf $d});
- $nok = 0
+ }
+ }
+ foreach my $t ("unionfs",'tmpfs') {
+ my $d = "$unionfs_dir/$t.$r.$union_id";
+ print {$run->{LOG}} "iurt: removing $d\n" if $run->{verbose} > 1;
+ if (system(qq{sudo rmdir $d})) {
+ print {$run->{LOG}} "ERROR iurt: removing $d failed ($!)\n";
+ return $union_id + 1
}
}
$union_id
@@ -881,7 +911,11 @@ sub clean_chroot_tmp {
sub check_mounted {
my ($mount_point, $type) = @_;
- open my $mount, '/proc/mounts' or die 'FATAL iurt: could not open /proc/mounts';
+ my $mount;
+ if (!open $mount, '/proc/mounts') {
+ print 'ERROR iurt: could not open /proc/mounts';
+ return
+ }
$mount_point =~ s,//+,/,g;
while (<$mount>) {
return 1 if /^\w+ $mount_point $type /
@@ -995,7 +1029,7 @@ sub dump_cache {
print {$run{LOG}} "ERROR iurt: manual file lock exist, do not save the cache\n";
} else {
open my $lock, ">$filename.lock";
- print $lock, $$;
+ print $lock $$;
close $lock;
unlink $filename;
link "$filename.tmp.$daydate", $filename;
@@ -1106,7 +1140,11 @@ sub perform_command {
print {$run{LOG}} "$command\n" if $run{verbose} > 2;
if ($opt{log}) {
#$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`;
- system("$command &> $logfile");
+ if ($opt{reopen_log}) {
+ system("$command 2>&1 >> $logfile");
+ } else {
+ system("$command >& $logfile");
+ }
} else {
$output = `$command 2>&1`;
}
@@ -1162,7 +1200,13 @@ sub perform_command {
if ($opt{log} && $config->{log_url}) {
$comment = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$comment|
}
- my $out = $output if length $output < 10000;
+ my $out;
+ if (length $output < 10000) {
+ $out = $output
+ } else {
+ $out = "Message too big, see http link for details\n"
+ }
+
if ($opt{mail} && $config->{sendmail} && !$config->{no_mail}{$opt{mail}}) {
if (! ($cache->{warning}{$opt{hash}}{$opt{mail}} % $opt{freq})) {
my $cc = join ',', grep { !$config->{no_mail}{$_} } split ',', $opt{cc};
@@ -1199,7 +1243,10 @@ sub check_chroot {
my ($chroot, $chroot_tar, $run) = @_;
print {$run{LOG}} "iurt: checking basesystem tar\n" if $run{verbose};
system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot" &> /dev/null});
- clean_chroot($chroot, $run, 1) or die "FATAL iurt: Could no prepare initial chroot";
+ if (!clean_chroot($chroot, $run, 1)) {
+ print "ERROR iurt: Could no prepare initial chroot";
+ return
+ }
perform_command("sudo $config->{install_chroot_binary} $config->{basesystem_media_root} $config->{basesystem_media_root}/media/$config->{basesystem_media} $chroot_tar $chroot 501 basesystem tar rpm-build rpm-mandriva-setup-build",
$run, $config,
mail => $config->{admin},
@@ -1400,7 +1447,7 @@ sub search_packages {
$maint{$srpm} = $mail;
print "iurt: will try to compile $srpm\n";
$to_compile++;
- push @{$run->{todo}}, [ $dir , $srpm ]
+ push @{$run->{todo}}, [ $dir , $srpm, 1 ]
}
foreach my $arch (@{$config->{supported_arch}}) {
$ok &&= $cache->{queue}{$srpm}
@@ -1431,16 +1478,20 @@ sub add_local_user {
# change the builder user to the local user id
# FIXME it seems that unionfs does not handle well the change of the uid of files
# if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) {
- if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) && system(qq|sudo chroot $chroot_tmp id $luser|)) {
- print {$run->{LOG}} "ERROR: setting userid to builder user in chroot failed, trying to check the chroot\n";
+ if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) || system(qq|sudo chroot $chroot_tmp id $luser|)) {
+ print {$run->{LOG}} "ERROR: setting userid $uid to $luser in $chroot_tmp failed, trying to check the chroot\n";
check_chroot($chroot, $chroot_tar, $run);
+ return
}
if (system(qq|sudo chroot $chroot_tmp cp -R /home/builder /home/$luser|)) {
- die "FATAL iurt: could not initialized $luser directory\n";
+ print "ERROR iurt: could not initialized $luser directory\n";
+ return
}
if (system(qq|sudo chroot $chroot_tmp chown -R $uid /home/$luser|)) {
- die "FATAL iurt: could not initialized $luser directory\n";
+ die "ERROR iurt: could not initialized $luser directory\n";
+ return
}
+ 1
}
sub create_temp_chroot {
@@ -1455,14 +1506,20 @@ sub create_temp_chroot {
my $tmpfs;
# we cannont just rm -rf $tmpfs, this create defunct processes afterwards (and lock particularly hard the urpmi database)
- $union_id = clean_unionfs($unionfs_dir, $run->{run}, $union_id);
+ $union_id = clean_unionfs($unionfs_dir, $run, $run->{run}, $union_id);
$tmpfs = "$unionfs_dir/tmpfs.$run->{run}.$union_id";
$chroot_tmp = "$unionfs_dir/unionfs.$run->{run}.$union_id";
if (!-d $tmpfs) {
- mkdir $tmpfs or die "FATAL iurt: Could not create $tmpfs ($!)"
+ if (!mkdir $tmpfs) {
+ print "ERROR iurt: Could not create $tmpfs ($!)";
+ return
+ }
}
if (! -d $chroot_tmp) {
- mkdir $chroot_tmp or die "FATAL iurt: Could not create $chroot_tmp ($!)"
+ if (!mkdir $chroot_tmp) {
+ print "ERROR iurt: Could not create $chroot_tmp ($!)";
+ return
+ }
}
if ($cache->{no_unionfs}{$srpm}) {
$unionfs_tmp = 0;
@@ -1471,15 +1528,24 @@ sub create_temp_chroot {
# if the previous package has been built without unionfs, chroot need to be cleaned
clean_chroot($chroot_tmp, $run) if !$unionfs_tmp;
$unionfs_tmp = 1;
- system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null}) and die "FATAL iurt: could not mount $tmpfs ($!)";
- system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run->{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|) and die "FATAL iurt: could not mount $tmpfs and $home/chroot_$run->{distro_tag}$debug_tag with unionfs ($!)";
- system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null") and die "FATAL iurt: could not mount /proc in the chroot $chroot_tmp.";
+ if (system(qq{sudo mount -t tmpfs none $tmpfs &>/dev/null})) {
+ print "ERROR iurt: could not mount $tmpfs ($!)";
+ return
+ }
+ if (system(qq|sudo mount -o dirs=$tmpfs=rw:$home/chroot_$run->{distro_tag}$debug_tag=ro -t unionfs none $chroot_tmp &>/dev/null|)) {
+ print "ERROR iurt: could not mount $tmpfs and $home/chroot_$run->{distro_tag}$debug_tag with unionfs ($!)";
+ return
+ }
+ if (system("sudo mount -t proc none $chroot_tmp/proc &>/dev/null")) {
+ print "ERROR iurt: could not mount /proc in the chroot $chroot_tmp.";
+ return
+ }
}
} else {
print {$run->{LOG}} "iurt: installing a new chroot in $chroot_tmp\n" if $run->{verbose} > 1;
clean_chroot($chroot_tmp, $run)
}
- $unionfs_tmp, $chroot_tmp
+ $union_id, $unionfs_tmp, $chroot_tmp
}
sub add_packages {
@@ -1540,11 +1606,164 @@ sub remove_chroot {
sub add_media {
my ($run, $config, $chroot) = @_;
print {$run->{LOG}} "iurt: adding media $run{chrooted_media} in chroot $chroot\n";
- perform_command("sudo chroot $chroot_tmp urpmi.addmedia --distrib $run{chrooted_media}",
+ if (!perform_command("sudo chroot $chroot urpmi.addmedia --distrib $run{chrooted_media}",
+ $run, $config,
+ mail => $config->{admin},
+ timeout => 120,
+ freq => 1,
+ retry => 5,
+ debug_mail => $run->{debug})) {
+ }
+ check_media_added($chroot, 'Main') or die "FATAL iurt: could not add media into the chroot"
+}
+
+sub add_local_media {
+ my ($run, $config, $chroot, $local_media) = @_;
+ print {$run->{LOG}} "iurt: adding media $local_media in chroot $chroot\n";
+ if (perform_command("sudo chroot $chroot urpmi.addmedia iurt_group file://$local_media",
$run, $config,
mail => $config->{admin},
timeout => 120,
freq => 1,
retry => 5,
- debug_mail => $run->{debug})
+ debug_mail => $run->{debug})) {
+ }
+ check_media_added($chroot, 'iurt_group') or die "FATAL iurt: could not add local media into the chroot"
+}
+
+sub check_media_added {
+ my ($chroot, $media) = @_;
+ my $medias = `sudo chroot $chroot urpmi.removemedia 2>&1`;
+ print "MEDIA $medias\n";
+ $medias =~ /one of.* $media/m
+}
+
+sub get_local_provides {
+ my ($run, $local_media) = @_;
+ opendir my $dir, $local_media;
+ my $urpm = new URPM;
+ foreach my $d (readdir $dir) {
+ $d =~ /\.rpm$/ or next;
+ my $id = $urpm->parse_rpm("$local_media/$d");
+ my $pkg = $urpm->{depslist}[$id];
+ foreach ($pkg->provides, $pkg->files) {
+ print {$run->{LOG}} "iurt: adding $_ as provides of $d\n" if $run->{verbose} > 2;
+ $run{local_provides}{$_} = $d
+ }
+ }
+}
+
+sub recreate_srpm {
+ my ($run, $config, $chroot_tmp, $dir, $srpm, $luser, $retry) = @_;
+# recreate a new srpm for buildarch condition in the spec file
+ print {$run->{LOG}} "iurt: copying $srpm to $chroot_tmp\n" if $run->{verbose} > 1;
+ perform_command("sudo cp $dir/$srpm $chroot_tmp/home/$luser/rpm/SRPMS/",
+ \%run, $config,
+ mail => $config->{admin},
+ error => "[REBUILD] cannot copy $srpm to $chroot_tmp",
+ debug_mail => $run->{debug},
+ hash => "copy_$srpm") or return;
+
+ my %opt = ( mail => $config->{admin},
+ error => "[REBUILD] cannot install $srpm in $chroot_tmp",
+ debug_mail => $run->{debug},
+ hash => "install_$srpm",
+ retry => $retry,
+ callback => sub {
+ my ($opt, $output) = @_;
+ print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug};
+ if ($output =~ /warning: group .* does not exist - using root/i) {
+ return 1
+ } elsif ($output =~ /user $luser does not exist/) {
+ print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1;
+ $opt->{error} = "[CHROOT] chroot is corrupted";
+ $opt->{retry} = 1 if !$opt->{retry};
+ return
+ }
+ });
+ print {$run->{LOG}} "iurt: recreating src.rpm...\n" if $run->{verbose};
+ if (!perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm -i /home/$luser/rpm/SRPMS/$srpm"},
+ \%run, $config, %opt)) {
+ print {$run->{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run->{debug};
+ if ($opt{retry}) {
+ check_chroot($chroot, $chroot_tar, \%run) or return;
+ return -1
+ }
+ return
+ }
+ # 20060515 This should not be necessairy any more if urpmi *.spec works, but it doesn't
+ #
+ perform_command(qq{sudo chroot $chroot_tmp su $luser -c "rpm --nodeps -bs /home/$luser/rpm/SPECS/*.spec"},
+ \%run, $config,
+ mail => $config->{admin},
+ error => "[REBUILD] cannot create $srpm in $chroot_tmp",
+ debug_mail => $run->{debug},
+ hash => "create_$srpm")
+}
+
+sub get_build_requires {
+ my ($run, $config) = @_;
+ $run{todo_requires} = {};
+ my ($union_id, $unionfs_tmp, $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or return;
+
+ my $urpm = new URPM;
+ foreach my $p (@{$run->{todo}}) {
+ my ($dir, $srpm, $s) = @$p;
+ recreate_srpm($run, $config, $chroot_tmp, $dir, $srpm, 'iurt') or return;
+ $s or next;
+ my $id = $urpm->parse_rpm("$dir/$srpm");
+ my $pkg = $urpm->{depslist}[$id];
+ foreach ($pkg->requires) {
+ print {$run->{LOG}} "iurt: adding $_ as requires of $srpm\n" if $run->{verbose} > 2;
+ $run{todo_requires}{$_} = $srpm
+ }
+ }
+}
+
+sub order_packages {
+ my ($run, $config, $provides, $local_media) = @_;
+ my @packages = @{$run{todo}};
+ my $move;
+ get_local_provides($run, $local_media) or return;
+ if (!$run{todo_requires}) {
+ get_build_requires($run, $config) or return
+ }
+ my %visit;
+ my %status;
+ do {
+ $move = 0;
+ foreach my $p (@packages) {
+ my ($dir, $rpm, $status) = @$p;
+ defined $status{$rpm} && $status{$rpm} == 0 and next;
+ print {$run->{LOG}} "iurt: checking packages $rpm\n";
+ foreach my $r (@{$run->{todo_requires}{$rpm}}) {
+ print {$run->{LOG}} "iurt: checking requires $r\n";
+ if (!$run->{local_provides}{$r}) {
+ if ($provides->{$r}) {
+ $status = 1
+ } else {
+ $status = 0
+ }
+ } elsif ($visit{$rpm}{$r}) {
+ # to evit loops
+ $status = 0
+ } elsif ($run->{done}{$rpm} && $run->{done}{$provides->{$r}}) {
+ if ($run->{done}{$rpm} < $run->{done}{$provides->{$r}}) {
+ $move = 1;
+ $status = $status{$provides->{$r}} + 1
+ } else {
+ $status = 0
+ }
+ } elsif ($status < $status{$provides->{$r}}) {
+ $move = 1;
+ $status = $status{$provides->{$r}} + 1
+ }
+ $visit{$rpm}{$r} = 1;
+ }
+ $status{$rpm} = $status;
+ $p->[2] = $status
+ }
+ } while ($move);
+ $run->{todo} = [ sort { $a->[2] <=> $b->[2] } @packages ];
+ @packages
}
generated by cgit v1.2.1 (git 2.21.0) at 2024-11-22 13:34:40 +0000