From c24757c923841de1901501b227d1411eafefc4e4 Mon Sep 17 00:00:00 2001 From: Pascal Terjan Date: Mon, 10 Dec 2012 20:50:42 +0000 Subject: Get rid of cache --- iurt | 139 ++++++++++++---------------------------------------- lib/Iurt/Chroot.pm | 1 - lib/Iurt/Config.pm | 85 -------------------------------- lib/Iurt/DKMS.pm | 5 +- lib/Iurt/Process.pm | 2 - lib/Iurt/Urpmi.pm | 5 +- 6 files changed, 34 insertions(+), 203 deletions(-) diff --git a/iurt b/iurt index 0bfdbcb..0c361fb 100755 --- a/iurt +++ b/iurt @@ -33,7 +33,7 @@ use strict; use RPM4::Header; -use Iurt::Config qw(config_usage get_date get_prefix config_init dump_cache_par get_maint check_arch %arch_comp get_package_prefix); +use Iurt::Config qw(config_usage get_date get_prefix config_init get_maint check_arch %arch_comp get_package_prefix); use Data::Dumper; use URPM; use Iurt::DKMS; @@ -91,10 +91,10 @@ $run{todo} = []; # # no_rsync, config_help and copy_srpm kept for compatibility reasons # - [ "", $program_name, 0, "[--cache] [--chrooted-urpmi ] [--concurrent-run] [--config foo value] [--warn] [--verbose integer] + [ "", $program_name, 0, "[--chrooted-urpmi ] [--config foo value] [--warn] [--verbose integer] [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}] [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] - [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--status] [--ignore-failure] + [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--status] [--repository ] [--rpmmacros [...]] {--config_help | --dkms {--media } @@ -137,9 +137,6 @@ $run{todo} = []; [ "a", "arch", 1, "", "Set the architecture", sub { ($run{my_arch}) = @_; 1 }, "Setting architecture" ], - [ "", "cache", 0, "", - "Use the global cache file", - sub { $run{use_cache} = 1 }, "Activating cache use" ], [ "", "copy-srpm", 0, "", "Copy also the regenerated SRPM", sub { $run{copy_srpm} = 1 }, "Activating the copy_srpm mode" ], @@ -170,9 +167,6 @@ $run{todo} = []; [ "", "clean", -1, " ... ", "Clean remaining chroot before runing", sub { $run{clean} = \@_ }, "Activating clean chroot flag" ], - [ "", "concurrent-run", 0, "", - "Allow several iurt to run on different machines (slower)", - sub { $run{concurrent_run} = 1 }, "Activating concurrent run checks" ], [ "", "parallel", 1, "", "Build up to packages in parallel", sub {($run{parallel}) = @_; 1}, "Enabling parallel build" ], @@ -194,9 +188,6 @@ $run{todo} = []; [ "g", "group", 0, "", "Activate group mode, packages will be compiled as a global set, not as individual packages", sub { $run{group} = 1 }, "Activating the group mode" ], - [ "", "ignore-failure", 0, "", - "Do not take into account the failure cache, try to recompile all the packages not synchronized", - sub { $run{ignore_failure} = 1 }, "Activating the mode ignoring previous failure" ], [ "l", "log", 1, "", "Log file.", sub { @@ -454,14 +445,10 @@ my %config_usage = ( default => 300, }, }, - cache_home => { - desc => 'Where to store the cache files', + pidfile_home => { + desc => 'Where to store the pidfile files', default => "$HOME/.bugs" }, - cache_min_size => { - desc => 'Minimal size to consider a cache file valid', - default => 1000000 - }, check_binary_file => { desc => 'Packages rebuild should be checked, however sometime rpm is segfaulting and the test is not correct', default => 0 @@ -598,14 +585,9 @@ if (!$lock && $run{dkms}) { } $run{lock} = $lock; -# cache file name is needed early to remove the manual lock file if the -# lock mechanism does not work - -mkpath $config->{cache_home}; -my $cachefile = "$config->{cache_home}/iurt.$run{distro_tag}.$run{my_arch}.$lock.cache"; -$run{cachefile} = $cachefile; if (!$run{debug} && $run{media} || $run{chroot}) { - $run{pidfile_home} = "$config->{cache_home}/"; + $run{pidfile_home} = $config->{pidfile_home}; + mkpath $run{pidfile_home}; $run{pidfile} = "iurt.$run{distro_tag}.$run{my_arch}.$lock"; check_pid(\%run); } @@ -630,43 +612,13 @@ if (!-d "$local_spool/log") { } $run{local_spool} = $local_spool; -my $cache; -my $clear_cache = 1; -if (-f $cachefile && $run{use_cache}) { - plog('INFO', "loading cache file $cachefile"); - - $cache = eval(cat_($cachefile)) - or plog('ERROR', "FATAL: could not load cache $cachefile ($!)"); - - if (!$cache) { - opendir my $cache_dir, $config->{cache_home}; - my $to_load; - - foreach my $file (readdir $cache_dir) { - (my $date) = $file =~ /iurt\.$run{distro_tag}\.$run{my_arch}\.$run{media}\.cache\.tmp\.(\d{8})/ or next; - if ($date > $to_load && -s "$config->{cache_home}/$file" > $config->{cache_min_size}) { - $to_load = $date; - $cachefile = "$config->{cache_home}/$file"; - } - } - - plog('NOTIFY', "loading alternate cache file $cachefile"); - $cache = eval(cat_($cachefile)) - or plog('ERROR', "FATAL: could not load cache $cachefile ($!)"); - } - $clear_cache = 0 if $cache; -} - -if ($clear_cache) { - $cache = { - rpm_srpm => {}, - failure => {}, - queue => {}, - warning => {}, - run => 1, - needed => {}, - }; -} +my $cache = { + rpm_srpm => {}, + queue => {}, + warning => {}, + run => 1, + needed => {}, +}; $run{cache} = $cache; my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint); @@ -683,18 +635,10 @@ if ($run{dkms}) { } $run{to_compile} = $to_compile; -dump_cache_par(\%run); - plog("Packages to build: $to_compile"); my ($fulldate, $daydate) = get_date(); -if ($run{use_cache}) { - $run{run} = $cache->{run}; - $run{run} ||= 1; - $cache->{run} = $run{run} + 1; -} else { - $run{run} = "0.$fulldate"; -} +$run{run} = "0.$fulldate"; $run{daydate} = $daydate; plog('DEBUG', "using $run{run} as chroot extension"); $run{user} ||= $ENV{USER}; @@ -803,8 +747,6 @@ if ($run{dkms} && $run{dkms_todo}) { # FIXME: (tv) kill this dead code or use it!! my $_s = sub { if ($run{main}) { - plog("dumping cache..."); - dump_cache_par(\%run); $Data::Dumper::Indent = 0; $Data::Dumper::Terse = 1; plog("Running environment:\n", Data::Dumper->Dump([\%run]), "\n"); @@ -831,18 +773,12 @@ sub rebuild_one { plog('NOTIFY', "Build package $srpm [$done/$to_compile]"); # When rebuilding all the media, src.rpm can be removed from mirror before we work on them unless (-f "$dir/$srpm") { - $cache->{failure}{$srpm} = 1; $run{status}{$srpm} = 'missing'; return; } # FIXME unfortunately urpmi stalls quite often my $retry = 0; - # current rpm is sometime segfaulting, and iurt is them blocked - # and cannot - # - # $cache->{failure}{$srpm} = 1; - # dump_cache(\%run); retry: $urpmi->clean_urpmi_process; @@ -888,7 +824,6 @@ retry: # This should eliminate bouncers that block the input queue # $srpm = $old_srpm; - $cache->{failure}{$srpm} = 1; $run{status}{$srpm} = 'recreate_srpm_failure'; return; } @@ -986,7 +921,6 @@ retry: }, freq => 1)) { - $cache->{failure}{$srpm} = 1; $run{status}{$srpm} = 'build_failure'; return; } @@ -1015,7 +949,6 @@ retry: } $run{status}{$srpm} = 'ok'; - delete $cache->{failure}{$srpm} if defined $cache->{failure}{$srpm}; if ($run{debug}) { plog("debug mode, skip other packages"); exit(); @@ -1086,7 +1019,6 @@ do { } elsif ($pid == 0) { #child $chroot_tmp .= "_" . int($i); rebuild_one($dir, $srpm, $status); - dump_cache_par(\%run); dump_status($local_spool, \%run); exit; } else { @@ -1094,7 +1026,6 @@ do { } } else { rebuild_one($dir, $srpm, $status); - dump_cache_par(\%run); dump_status($local_spool, \%run); } } @@ -1135,8 +1066,6 @@ if (!$run{debug} && !$run{use_old_chroot}) { plog("reprocess generated packages queue"); process_queue($config, \%run, \@wrong_rpm); -dump_cache_par(\%run); - plog('FAIL', "ERROR: RPM with a wrong SRPM name") if @wrong_rpm; if (@wrong_rpm && open my $file, ">$local_spool/log/wrong_srpm_names.log") { foreach (@wrong_rpm) { @@ -1268,24 +1197,22 @@ sub check_pid { close $lock2; } } - if (!$run->{concurrent_run}) { - opendir my $dir, $run->{pidfile_home}; - foreach my $f (readdir $dir) { - my ($pid_host) = $f =~ /$pidfile\.pid\.(.*)\.pid$/ or next; - if ($pid_host ne $hostname) { - my $pf = "$run->{pidfile_home}/$f"; - open my $test_PID, $pf; - my $pid = <$test_PID>; - my (@stat) = stat $pf; - my $time = $stat[9]; - my $diff = time()-$time; - my $msg = "$program_name: an other iurt is running for $run->{my_arch} on $pid_host, pid $pid, since $diff seconds"; - if ($diff < 36000) { - plog("$msg\n"); - exit(); - } else { - plog("$msg, ignoring it"); - } + opendir my $dir, $run->{pidfile_home}; + foreach my $f (readdir $dir) { + my ($pid_host) = $f =~ /$pidfile\.pid\.(.*)\.pid$/ or next; + if ($pid_host ne $hostname) { + my $pf = "$run->{pidfile_home}/$f"; + open my $test_PID, $pf; + my $pid = <$test_PID>; + my (@stat) = stat $pf; + my $time = $stat[9]; + my $diff = time()-$time; + my $msg = "$program_name: an other iurt is running for $run->{my_arch} on $pid_host, pid $pid, since $diff seconds"; + if ($diff < 36000) { + plog("$msg\n"); + exit(); + } else { + plog("$msg, ignoring it"); } } } @@ -1411,10 +1338,6 @@ sub search_packages { if ($config->{unwanted_packages} && $srpm =~ /$config->{unwanted_packages}/) { next } my $ok = 1; if (check_version($run, $srpm, $srpm_version)) { - if (!$run->{ignore_failure} && defined $cache->{failure}{$srpm}) { - $run->{status}{$srpm} = 'build_failure'; - next; - } my $check_needed = check_needed($srpm, $cache, $provides); $run->{status}{$srpm} = 'missing_buildrequires' if !$check_needed; -f "$dir/$srpm" or next; diff --git a/lib/Iurt/Chroot.pm b/lib/Iurt/Chroot.pm index 3995d21..447ef23 100644 --- a/lib/Iurt/Chroot.pm +++ b/lib/Iurt/Chroot.pm @@ -5,7 +5,6 @@ use base qw(Exporter); use MDV::Distribconf::Build; use MDK::Common; use Iurt::Process qw(sudo); -use Iurt::Config qw(dump_cache_par); use Iurt::Util qw(plog); # perl_checker: use Iurt::Urpmi use File::Temp 'mktemp'; diff --git a/lib/Iurt/Config.pm b/lib/Iurt/Config.pm index 173ed9e..c793cba 100644 --- a/lib/Iurt/Config.pm +++ b/lib/Iurt/Config.pm @@ -13,8 +13,6 @@ our @EXPORT = qw( config_usage config_init get_date - dump_cache - dump_cache_par get_maint get_date get_prefix @@ -123,89 +121,6 @@ sub get_package_prefix { "$prefix1$prefix2"; } -=head2 dump_cache($run, $config) - -Create a string based on the current date and time -I<%run> is the current running options -Dump the cache - -=cut - -sub dump_cache { - my ($run) = @_; - my $program_name = $run->{program_name}; - my $filename = $run->{cachefile}; - my $cache = $run->{cache}; - my $daydate = $run->{daydate}; - open my $file, ">$filename.tmp.$daydate" or die "FATAL $program_name dump_cache: cannot open $filename.tmp"; - $Data::Dumper::Indent = 1; - $Data::Dumper::Terse = 1; - print $file Data::Dumper->Dump([ $cache ], [ "cache" ]); - # flock does not work on network files and lockf seems to fail too - plog('DEBUG', "locking to dump the cache in $filename"); - if (-f "$filename.lock") { - plog('ERROR', 'ERROR: manual file lock exist, do not save the cache'); - } else { - open my $lock, ">$filename.lock"; - print $lock $$; - close $lock; - unlink $filename; - link "$filename.tmp.$daydate", $filename; - unlink "$filename.lock"; - } -} - -# FIXME need to merge with the simpler dump_cache -sub dump_cache_par { - my ($run) = @_; - my $filename = $run->{cachefile}; - my $cache = $run->{cache}; - my $daydate = $run->{daydate}; - - # Right now there are no mechanism of concurrent access/write to the cache. There is - # on global lock for one iurt session. A finer cache access would allow several iurt running - # but the idea is more to have a global parrallel build than several local ones. - return if $run->{debug} || !$run->{use_cache}; - open my $file, ">$filename.tmp.$daydate" or die "FATAL iurt dump_cache: cannot open $filename.tmp"; - if ($run->{concurrent_run}) { - plog('DEBUG', "merging cache"); - my $old_cache; - if (-f $filename) { - plog('DEBUG', "loading cache file $filename"); - $old_cache = eval(cat_($filename)); - - foreach my $k ('rpm_srpm', 'failure', 'queue', 'needed', 'warning', 'buildrequires') { - foreach my $rpm (%{$old_cache->{$k}}) { - $cache->{$k}{$rpm} ||= $old_cache->{$k}{$rpm}; - } - } - } - # $cache = { rpm_srpm => {}, failure => {}, queue => {}, warning => {}, run => 1, needed => {} } - } - $Data::Dumper::Indent = 1; - $Data::Dumper::Terse = 1; - print $file Data::Dumper->Dump([ $cache ], [ "cache" ]); - # flock does not work on network files and lockf seems to fail too - my $status = 1; #File::lockf::lock($file); - if (!$status) { - unlink $filename; - link "$filename.tmp.$daydate", $filename; - File::lockf::ulock($file); - } else { - plog('WARN', "WARNING: locking the cache file $filename failed (status $status $!), try to lock manually"); - if (-f "$filename.lock") { - plog('ERROR', "ERROR: manual file lock exist, do not save the cache"); - } else { - open my $lock, ">$filename.lock"; - print $lock $$; - close $lock; - unlink $filename; - link "$filename.tmp.$daydate", $filename; - unlink "$filename.lock"; - } - } -} - sub get_maint { my ($run, $srpm) = @_; my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.[^.]+$/; diff --git a/lib/Iurt/DKMS.pm b/lib/Iurt/DKMS.pm index 82bcdc3..d7d6d2f 100644 --- a/lib/Iurt/DKMS.pm +++ b/lib/Iurt/DKMS.pm @@ -4,7 +4,7 @@ use strict; use base qw(Exporter); use MDV::Distribconf::Build; use Iurt::Chroot qw(clean_and_build_chroot add_local_user dump_rpmmacros); -use Iurt::Config qw(get_maint get_prefix dump_cache); +use Iurt::Config qw(get_maint get_prefix); use Iurt::Mail qw(sendmail); use File::NCopy qw(copy); use Iurt::Process qw(sudo); @@ -120,7 +120,7 @@ sub search_dkms { foreach my $k (@kernel) { my ($kernel, $modulesdir) = @$k; plog("checking $realmodule-kernel-$modulesdir-$realversion"); - next if $cache->{dkms}{"$realmodule-kernel-$modulesdir-$realversion"} && !$run->{ignore_failure}; + next if $cache->{dkms}{"$realmodule-kernel-$modulesdir-$realversion"}; if (!$modules{$realmodule}{$realversion}{$modulesdir}) { my (@choices); if (my ($prefix, $v) = $kernel =~ /^(.*?-?)(2\..*)/) { @@ -293,7 +293,6 @@ sub dkms_compile { process_dkms_queue($self, 0, 0, $media, "$dkms_spool/$media"); # compile dkms modules } - dump_cache($run); $done; } # FIXME will replace the iurt process_qeue when youri-queue is active diff --git a/lib/Iurt/Process.pm b/lib/Iurt/Process.pm index d05e4c4..66b604f 100644 --- a/lib/Iurt/Process.pm +++ b/lib/Iurt/Process.pm @@ -5,7 +5,6 @@ use base qw(Exporter); use MDK::Common; use Filesys::Df qw(df); use Iurt::Mail qw(sendmail); -use Iurt::Config qw(dump_cache_par); use Iurt::Util qw(plog); use POSIX ":sys_wait_h"; use Sys::Load qw(getload); @@ -161,7 +160,6 @@ sub handle_command_error { plog('INFO', "--------------- end of command output ---------------"); if ($opt{die}) { - dump_cache_par($run); die "FATAL: $opt{error}."; } } diff --git a/lib/Iurt/Urpmi.pm b/lib/Iurt/Urpmi.pm index 96c9d96..8ecee8c 100644 --- a/lib/Iurt/Urpmi.pm +++ b/lib/Iurt/Urpmi.pm @@ -7,7 +7,7 @@ use File::NCopy qw(copy); use MDV::Distribconf::Build; use Iurt::Chroot qw(add_local_user create_temp_chroot); use Iurt::Process qw(perform_command clean_process sudo); -use Iurt::Config qw(dump_cache_par get_maint get_package_prefix); +use Iurt::Config qw(get_maint get_package_prefix); use Iurt::Util qw(plog); use urpm; @@ -404,7 +404,6 @@ sub install_packages_old { wait_regexp => { 'database locked' => \&wait_urpmi }, error_regexp => 'unable to access', log => $log_spool)) { - $cache->{failure}{$srpm} = 1; $run->{status}{$srpm} = 'binary_test_failure'; return 0; } @@ -541,7 +540,6 @@ sub install_packages { )) { plog('DEBUG', "urpmi command failed."); if (!clean_process("$self->{urpmi_command} @to_install")) { - dump_cache_par($run); die "FATAL $program_name: Could not have urpmi working !"; } $ok = 0; @@ -565,7 +563,6 @@ sub clean_urpmi_process { if (!$run->{chrooted_urpmi}) { my $match = $self->{urpmi_command} or return; if (!clean_process($match)) { - dump_cache_par($run); die "FATAL $program_name: Could not have urpmi working !"; } } -- cgit v1.2.1