diff options
| -rwxr-xr-x | emi | 67 | ||||
| -rw-r--r-- | lib/Iurt/File.pm | 161 | ||||
| -rwxr-xr-x | ulri | 99 |
3 files changed, 143 insertions, 184 deletions
@@ -32,7 +32,7 @@ use File::Path qw(make_path); use Iurt::Config qw(config_usage config_init get_author_email get_mandatory_arch); use Iurt::Process qw(check_pid); use Iurt::Mail qw(sendmail); -use Iurt::File qw(check_upload_tree); +use Iurt::File qw(get_upload_tree_state); use Iurt::Util qw(plog_init plog); use Data::Dumper; use MDK::Common qw(cat_ if_ find touch); @@ -118,70 +118,7 @@ my $todo = "$config->{queue}/todo"; my $done = "$config->{queue}/done"; my $reject = "$config->{queue}/rejected"; -my %pkg_tree; - - -# -# Gather data from upload tree -# - -sub done_func { - my ($_todo, $_f, $m, $s, $r) = @_; - - my $media = "$m/$s"; - - if ($r =~ /^(\d{14}\.\w+\.\w+\.\d+)([_.].+)$/) { - my ($prefix, $suffix) = ($1, $2); - if ($suffix =~ /^_(\w+)\.(\w+)$/) { - my ($arch, $result) = ($1, $2); - if ($result eq 'done') { - plog('DEBUG', "found .done ($prefix) for $arch"); - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; - } elsif ($result eq 'excluded') { - $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; - plog('DEBUG', "found .excluded ($prefix) for $arch"); - $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1; - } - } elsif ($suffix =~ /^\.(\w+)$/) { - my $action = $1; - if ($action eq 'upload') { - plog('DEBUG', "found already uploaded ($prefix)"); - $pkg_tree{$prefix}{media}{$media}{uploaded} = 1; - } - } - } -} - -sub done_post { - my ($_todo, $f, $m, $s, $r) = @_; - my $media = "$m/$s"; - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { - my ($prefix, $rpm, $arch) = ($1, $2, $3); - $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; - plog('DEBUG', "found rpm $rpm ($prefix) for media $media"); - $pkg_tree{$prefix}{target} = $f; - $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; - if ($arch eq 'src') { - push @{$pkg_tree{$prefix}{media}{$media}{srpms}}, $rpm; - $pkg_tree{$prefix}{media}{$media}{arch}{src} = 1; - } - push @{$pkg_tree{$prefix}{media}{$media}{rpms}} , $rpm; - } -} - -sub todo_func { - my ($_todo, $_f, $_m, $_s, $r) = @_; - - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { - my ($prefix, $rpm) = ($1, $2); - plog('DEBUG', "found todo rpm $rpm ($prefix)"); - push @{$pkg_tree{$prefix}{srpms}}, $rpm; - } -} - -check_upload_tree($done, \&done_func, \&done_post); -check_upload_tree($todo, \&todo_func); - +my %pkg_tree = get_upload_tree_state($config); # # Decide what should be uploaded diff --git a/lib/Iurt/File.pm b/lib/Iurt/File.pm index b91ec2c..33be6a1 100644 --- a/lib/Iurt/File.pm +++ b/lib/Iurt/File.pm @@ -5,55 +5,45 @@ use Iurt::Config qw(get_mandatory_arch); use File::Copy 'move'; use File::Path 'make_path'; use Iurt::Util qw(plog); -use MDK::Common qw(find member partition); +use MDK::Common qw(cat_ find member partition); use strict; our @EXPORT = qw( - check_upload_tree + get_upload_tree_state cleanup_failed_build ); -sub check_upload_tree { - my ($todo, $func, $o_post) = @_; +sub apply_to_upload_tree { + my ($tree_root, $func) = @_; # Squash double slashes for cosmetics - $todo =~ s!/+!/!g; + $tree_root =~ s!/+!/!g; - opendir(my $dir, $todo); - plog('INFO', "check dir: $todo"); + opendir(my $dir, $tree_root); + plog('INFO', "check dir: $tree_root"); foreach my $f (readdir $dir) { $f =~ /^\.{1,2}$/ and next; - if (-d "$todo/$f") { - plog('DEBUG', "checking target $todo/$f"); - opendir my $target_dir, "$todo/$f"; + if (-d "$tree_root/$f") { + plog('DEBUG', "checking target $tree_root/$f"); + opendir my $target_dir, "$tree_root/$f"; foreach my $m (readdir $target_dir) { $m =~ /^\.{1,2}$/ and next; - if (-d "$todo/$f/$m") { - plog('DEBUG', "checking media $todo/$f/$m"); - opendir my $media_dir, "$todo/$f/$m"; + if (-d "$tree_root/$f/$m") { + plog('DEBUG', "checking media $tree_root/$f/$m"); + opendir my $media_dir, "$tree_root/$f/$m"; foreach my $s (readdir $media_dir) { $s =~ /^\.{1,2}$/ and next; - if (-d "$todo/$f/$m/$s") { + if (-d "$tree_root/$f/$m/$s") { if ($func) { - opendir my $submedia_dir, "$todo/$f/$m/$s"; + opendir my $submedia_dir, "$tree_root/$f/$m/$s"; foreach my $r (readdir $submedia_dir) { $r =~ /^\.{1,2}$/ and next; - $func->($todo, $f, $m, $s, $r); + $func->($tree_root, $f, $m, $s, $r); } } - # cleaning - if ($o_post) { - opendir my $submedia_dir, "$todo/$f/$m/$s"; - foreach my $r (readdir $submedia_dir) { - $r =~ /^\.{1,2}$/ and next; - $o_post->($todo, $f, $m, $s, $r); - } - } - } else { - # may need to check also here for old target } } } @@ -113,3 +103,122 @@ sub cleanup_failed_build { } } +sub get_upload_tree_state { + our ($config) = @_; + + our %pkg_tree; + my $todo = "$config->{queue}/todo"; + my $done = "$config->{queue}/done"; + + sub todo_func { + my ($todo, $f, $m, $s, $r) = @_; + + my $media = "$m/$s"; + + if ($r =~ /(\d{14}\.(\w+)\.\w+\.\d+)_(.*\.src\.rpm)$/) { + my ($prefix, $user, $srpm) = ($1, $2, $3); + + plog('DEBUG', "found srpm $srpm ($prefix)"); + $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; + $pkg_tree{$prefix}{target} = $f; + $pkg_tree{$prefix}{user} = $user; + push @{$pkg_tree{$prefix}{srpms}} , $srpm; + my ($name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/; + + $pkg_tree{$prefix}{srpm_name}{$name} = $srpm; + } + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) { + my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6); + + # Set path here too has we may have a lock without the src.rpm + $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; + + $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; + plog('DEBUG', "found lock on $host/$arch for $prefix"); + + if ($arch =~ /noarch/) { + plog('DEBUG', "... and $prefix is noarch"); + $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1; + $arch =~ s/-.*//; + } + + $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; + + my $time = read_line("$todo/$f/$m/$s/$r"); + $time = (split ' ', $time)[2]; + push @{$pkg_tree{$prefix}{media}{$media}{bot}}, { + bot => $bot, + host => $host, + date => $date, + pid => $pid, + 'arch' => $arch, + 'time' => $time + }; + } + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) { + my $prefix = $1; + my @deps = map { chomp(); $_ } cat_("$todo/$f/$m/$s/$r"); + plog('DEBUG', "Adding dependency $_ ($prefix)") foreach @deps; + + $pkg_tree{$prefix}{deps} = \@deps; + } + } + + sub done_func { + my ($_todo, $f, $m, $s, $r) = @_; + + my $media = "$m/$s"; + + if ($r =~ /^(\d{14}\.\w+\.\w+\.\d+)([_.].+)$/) { + my ($prefix, $suffix) = ($1, $2); + $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; + if ($suffix =~ /^_(.*\.([^.]+)\.rpm)$/) { + my ($rpm, $arch) = ($1, $2); + $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; + plog('DEBUG', "found already built rpm $rpm ($prefix) for media $media"); + $pkg_tree{$prefix}{target} = $f; + if ($arch eq 'src') { + $pkg_tree{$prefix}{media}{$media}{arch}{src} = 1; + } + push @{$pkg_tree{$prefix}{media}{$media}{rpms}} , $rpm; + push @{$pkg_tree{$prefix}{rpms}} , $rpm; + } elsif ($suffix =~ /^_(\w+)\.(\w+)$/) { + my ($arch, $result) = ($1, $2); + plog('DEBUG', "found .$result ($prefix) for $arch"); + if ($result eq 'done') { + $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; + } elsif ($result eq 'excluded') { + $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; + $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1; + } elsif ($result eq 'fail') { + $pkg_tree{$prefix}{media}{$media}{failed_arch}{$arch} = 1; + } else { + plog('WARNING', "unknown state $arch.$result for $prefix"); + } + } elsif ($suffix =~ /^\.(\w+)$/) { + my $action = $1; + if ($action eq 'upload') { + plog('DEBUG', "found already uploaded ($prefix)"); + $pkg_tree{$prefix}{media}{$media}{uploaded} = 1; + } + } + } + } + + apply_to_upload_tree($done, \&done_func); + apply_to_upload_tree($todo, \&todo_func); + + return %pkg_tree; +} + +sub read_line { + my $file = shift; + + open my $FILE, "<$file" or die "FATAL: can't open $file for reading"; + my $contents = <$FILE>; + + $contents; +} + @@ -35,7 +35,7 @@ use strict; use MDK::Common qw(any cat_ if_ find); use Iurt::Config qw(config_usage get_date config_init get_author_email check_arch check_noarch); use Iurt::Process qw(check_pid); -use Iurt::File qw(check_upload_tree cleanup_failed_build); +use Iurt::File qw(cleanup_failed_build get_upload_tree_state); use Iurt::Mail qw(sendmail); use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput); use File::Copy 'move'; @@ -204,8 +204,6 @@ if ($df->{per} == 100) { ($fulldate, $daydate) = get_date(); -my %pkg_tree; - my $todo = "$config->{queue}/todo"; my $failure = "$config->{queue}/failure"; my $done = "$config->{queue}/done"; @@ -219,78 +217,7 @@ plog('MSG', "check uploads tree"); plog('DEBUG', "input queue is $todo"); -sub todo_func { - my ($todo, $f, $m, $s, $r) = @_; - - my $media = "$m/$s"; - - if ($r =~ /(\d{14}\.(\w+)\.\w+\.\d+)_(.*\.src\.rpm)$/) { - my ($prefix, $user, $srpm) = ($1, $2, $3); - - plog('DEBUG', "found srpm $srpm ($prefix)"); - $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; - $pkg_tree{$prefix}{target} = $f; - $pkg_tree{$prefix}{user} = $user; - push @{$pkg_tree{$prefix}{srpms}} , $srpm; - my ($name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/; - - return $pkg_tree{$prefix}{srpm_name}{$name} = $srpm; - } - - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) { - my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6); - - # Set path here too has we may have a lock without the src.rpm - $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s"; - - $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; - plog('DEBUG', "found lock on $host/$arch for $prefix"); - - if ($arch =~ /noarch/) { - plog('DEBUG', "... and $prefix is noarch"); - $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1; - $arch =~ s/-.*//; - } - - # this should be in the cache, but waiting for a cache-clean option - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; - - my $time = read_line("$todo/$f/$m/$s/$r"); - $time = (split ' ', $time)[2]; - push @{$pkg_tree{$prefix}{media}{$media}{bot}}, { - bot => $bot, - host => $host, - date => $date, - pid => $pid, - 'arch' => $arch, - 'time' => $time - }; - } - - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) { - my $prefix = $1; - my @deps = map { chomp(); $_ } cat_("$todo/$f/$m/$s/$r"); - plog('DEBUG', "Adding dependency $_ ($prefix)") foreach @deps; - - $pkg_tree{$prefix}{deps} = \@deps; - } -} - -sub done_func { - my ($_todo, $_f, $m, $s, $r) = @_; - - my $media = "$m/$s"; - - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.(done|fail|excluded)$/) { - my ($prefix, $arch) = ($1, $2); - $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch}; - $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1; - } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { - my ($prefix, $rpm) = ($1, $2); - plog('DEBUG', "found already built rpm $rpm ($prefix)"); - push @{$pkg_tree{$prefix}{rpms}} , $rpm; - } -} +my %pkg_tree = get_upload_tree_state($config); sub exclude_machine { my ($config, $host) = @_; @@ -300,15 +227,8 @@ sub exclude_machine { } } -check_upload_tree($todo, \&todo_func); - -# getting already compiled packages -# The cache should not be needed if the .done file are removed as the same -# time as the src.rpm in the todo tree -check_upload_tree($done, \&done_func); - # Load bot status - +# foreach my $prefix (keys %pkg_tree) { my $ent = $pkg_tree{$prefix}; foreach my $media (keys %{$ent->{media}}) { @@ -586,9 +506,10 @@ foreach my $prefix (sort keys %pkg_tree) { foreach my $arch (@arch_list) { # Skip this arch if package is already building as noarch or for this arch - # + # or if it should not be built on this arch next if $pkg_tree{$prefix}{media}{$media}{arch}{noarch}; next if $pkg_tree{$prefix}{media}{$media}{arch}{$arch}; + next if $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch}; # If all packages in a group are noarch, consider the entire group # as noarch @@ -603,6 +524,7 @@ foreach my $prefix (sort keys %pkg_tree) { plog('WARN', "excluding from $arch: $excluded"); create_file("$done_dir/${prefix}_$arch.excluded", "ulri $arch excluded"); + $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1; next; } @@ -830,15 +752,6 @@ sub create_file { print $FILE "@contents"; } -sub read_line { - my $file = shift; - - open my $FILE, "<$file" or die "FATAL: can't open $file for reading"; - my $contents = <$FILE>; - - $contents; -} - sub check_file_timeout { my ($file, $time) = @_; |