aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPascal Terjan <pterjan@gmail.com>2016-01-23 18:47:25 +0000
committerPascal Terjan <pterjan@gmail.com>2016-01-23 23:24:53 +0000
commit90dae21c83006471b2123e0ba138f260d6224466 (patch)
treedc7199347fbcbf2f26c986bb30c0e96e13db3616
parent84174a38a9b094c3a5cbef07a48b84b9c65d5bf8 (diff)
downloadiurt-90dae21c83006471b2123e0ba138f260d6224466.tar
iurt-90dae21c83006471b2123e0ba138f260d6224466.tar.gz
iurt-90dae21c83006471b2123e0ba138f260d6224466.tar.bz2
iurt-90dae21c83006471b2123e0ba138f260d6224466.tar.xz
iurt-90dae21c83006471b2123e0ba138f260d6224466.zip
Use a shared function to load the state of upload tree
-rwxr-xr-xemi67
-rw-r--r--lib/Iurt/File.pm161
-rwxr-xr-xulri99
3 files changed, 143 insertions, 184 deletions
diff --git a/emi b/emi
index 750df44..d6f1b6d 100755
--- a/emi
+++ b/emi
@@ -32,7 +32,7 @@ use File::Path qw(make_path);
use Iurt::Config qw(config_usage config_init get_author_email get_mandatory_arch);
use Iurt::Process qw(check_pid);
use Iurt::Mail qw(sendmail);
-use Iurt::File qw(check_upload_tree);
+use Iurt::File qw(get_upload_tree_state);
use Iurt::Util qw(plog_init plog);
use Data::Dumper;
use MDK::Common qw(cat_ if_ find touch);
@@ -118,70 +118,7 @@ my $todo = "$config->{queue}/todo";
my $done = "$config->{queue}/done";
my $reject = "$config->{queue}/rejected";
-my %pkg_tree;
-
-
-#
-# Gather data from upload tree
-#
-
-sub done_func {
- my ($_todo, $_f, $m, $s, $r) = @_;
-
- my $media = "$m/$s";
-
- if ($r =~ /^(\d{14}\.\w+\.\w+\.\d+)([_.].+)$/) {
- my ($prefix, $suffix) = ($1, $2);
- if ($suffix =~ /^_(\w+)\.(\w+)$/) {
- my ($arch, $result) = ($1, $2);
- if ($result eq 'done') {
- plog('DEBUG', "found .done ($prefix) for $arch");
- $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
- } elsif ($result eq 'excluded') {
- $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
- plog('DEBUG', "found .excluded ($prefix) for $arch");
- $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1;
- }
- } elsif ($suffix =~ /^\.(\w+)$/) {
- my $action = $1;
- if ($action eq 'upload') {
- plog('DEBUG', "found already uploaded ($prefix)");
- $pkg_tree{$prefix}{media}{$media}{uploaded} = 1;
- }
- }
- }
-}
-
-sub done_post {
- my ($_todo, $f, $m, $s, $r) = @_;
- my $media = "$m/$s";
- if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
- my ($prefix, $rpm, $arch) = ($1, $2, $3);
- $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
- plog('DEBUG', "found rpm $rpm ($prefix) for media $media");
- $pkg_tree{$prefix}{target} = $f;
- $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
- if ($arch eq 'src') {
- push @{$pkg_tree{$prefix}{media}{$media}{srpms}}, $rpm;
- $pkg_tree{$prefix}{media}{$media}{arch}{src} = 1;
- }
- push @{$pkg_tree{$prefix}{media}{$media}{rpms}} , $rpm;
- }
-}
-
-sub todo_func {
- my ($_todo, $_f, $_m, $_s, $r) = @_;
-
- if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
- my ($prefix, $rpm) = ($1, $2);
- plog('DEBUG', "found todo rpm $rpm ($prefix)");
- push @{$pkg_tree{$prefix}{srpms}}, $rpm;
- }
-}
-
-check_upload_tree($done, \&done_func, \&done_post);
-check_upload_tree($todo, \&todo_func);
-
+my %pkg_tree = get_upload_tree_state($config);
#
# Decide what should be uploaded
diff --git a/lib/Iurt/File.pm b/lib/Iurt/File.pm
index b91ec2c..33be6a1 100644
--- a/lib/Iurt/File.pm
+++ b/lib/Iurt/File.pm
@@ -5,55 +5,45 @@ use Iurt::Config qw(get_mandatory_arch);
use File::Copy 'move';
use File::Path 'make_path';
use Iurt::Util qw(plog);
-use MDK::Common qw(find member partition);
+use MDK::Common qw(cat_ find member partition);
use strict;
our @EXPORT = qw(
- check_upload_tree
+ get_upload_tree_state
cleanup_failed_build
);
-sub check_upload_tree {
- my ($todo, $func, $o_post) = @_;
+sub apply_to_upload_tree {
+ my ($tree_root, $func) = @_;
# Squash double slashes for cosmetics
- $todo =~ s!/+!/!g;
+ $tree_root =~ s!/+!/!g;
- opendir(my $dir, $todo);
- plog('INFO', "check dir: $todo");
+ opendir(my $dir, $tree_root);
+ plog('INFO', "check dir: $tree_root");
foreach my $f (readdir $dir) {
$f =~ /^\.{1,2}$/ and next;
- if (-d "$todo/$f") {
- plog('DEBUG', "checking target $todo/$f");
- opendir my $target_dir, "$todo/$f";
+ if (-d "$tree_root/$f") {
+ plog('DEBUG', "checking target $tree_root/$f");
+ opendir my $target_dir, "$tree_root/$f";
foreach my $m (readdir $target_dir) {
$m =~ /^\.{1,2}$/ and next;
- if (-d "$todo/$f/$m") {
- plog('DEBUG', "checking media $todo/$f/$m");
- opendir my $media_dir, "$todo/$f/$m";
+ if (-d "$tree_root/$f/$m") {
+ plog('DEBUG', "checking media $tree_root/$f/$m");
+ opendir my $media_dir, "$tree_root/$f/$m";
foreach my $s (readdir $media_dir) {
$s =~ /^\.{1,2}$/ and next;
- if (-d "$todo/$f/$m/$s") {
+ if (-d "$tree_root/$f/$m/$s") {
if ($func) {
- opendir my $submedia_dir, "$todo/$f/$m/$s";
+ opendir my $submedia_dir, "$tree_root/$f/$m/$s";
foreach my $r (readdir $submedia_dir) {
$r =~ /^\.{1,2}$/ and next;
- $func->($todo, $f, $m, $s, $r);
+ $func->($tree_root, $f, $m, $s, $r);
}
}
- # cleaning
- if ($o_post) {
- opendir my $submedia_dir, "$todo/$f/$m/$s";
- foreach my $r (readdir $submedia_dir) {
- $r =~ /^\.{1,2}$/ and next;
- $o_post->($todo, $f, $m, $s, $r);
- }
- }
- } else {
- # may need to check also here for old target
}
}
}
@@ -113,3 +103,122 @@ sub cleanup_failed_build {
}
}
+sub get_upload_tree_state {
+ our ($config) = @_;
+
+ our %pkg_tree;
+ my $todo = "$config->{queue}/todo";
+ my $done = "$config->{queue}/done";
+
+ sub todo_func {
+ my ($todo, $f, $m, $s, $r) = @_;
+
+ my $media = "$m/$s";
+
+ if ($r =~ /(\d{14}\.(\w+)\.\w+\.\d+)_(.*\.src\.rpm)$/) {
+ my ($prefix, $user, $srpm) = ($1, $2, $3);
+
+ plog('DEBUG', "found srpm $srpm ($prefix)");
+ $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
+ $pkg_tree{$prefix}{target} = $f;
+ $pkg_tree{$prefix}{user} = $user;
+ push @{$pkg_tree{$prefix}{srpms}} , $srpm;
+ my ($name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/;
+
+ $pkg_tree{$prefix}{srpm_name}{$name} = $srpm;
+ }
+
+ if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) {
+ my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6);
+
+ # Set path here too has we may have a lock without the src.rpm
+ $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
+
+ $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
+ plog('DEBUG', "found lock on $host/$arch for $prefix");
+
+ if ($arch =~ /noarch/) {
+ plog('DEBUG', "... and $prefix is noarch");
+ $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1;
+ $arch =~ s/-.*//;
+ }
+
+ $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
+
+ my $time = read_line("$todo/$f/$m/$s/$r");
+ $time = (split ' ', $time)[2];
+ push @{$pkg_tree{$prefix}{media}{$media}{bot}}, {
+ bot => $bot,
+ host => $host,
+ date => $date,
+ pid => $pid,
+ 'arch' => $arch,
+ 'time' => $time
+ };
+ }
+
+ if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) {
+ my $prefix = $1;
+ my @deps = map { chomp(); $_ } cat_("$todo/$f/$m/$s/$r");
+ plog('DEBUG', "Adding dependency $_ ($prefix)") foreach @deps;
+
+ $pkg_tree{$prefix}{deps} = \@deps;
+ }
+ }
+
+ sub done_func {
+ my ($_todo, $f, $m, $s, $r) = @_;
+
+ my $media = "$m/$s";
+
+ if ($r =~ /^(\d{14}\.\w+\.\w+\.\d+)([_.].+)$/) {
+ my ($prefix, $suffix) = ($1, $2);
+ $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
+ if ($suffix =~ /^_(.*\.([^.]+)\.rpm)$/) {
+ my ($rpm, $arch) = ($1, $2);
+ $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
+ plog('DEBUG', "found already built rpm $rpm ($prefix) for media $media");
+ $pkg_tree{$prefix}{target} = $f;
+ if ($arch eq 'src') {
+ $pkg_tree{$prefix}{media}{$media}{arch}{src} = 1;
+ }
+ push @{$pkg_tree{$prefix}{media}{$media}{rpms}} , $rpm;
+ push @{$pkg_tree{$prefix}{rpms}} , $rpm;
+ } elsif ($suffix =~ /^_(\w+)\.(\w+)$/) {
+ my ($arch, $result) = ($1, $2);
+ plog('DEBUG', "found .$result ($prefix) for $arch");
+ if ($result eq 'done') {
+ $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
+ } elsif ($result eq 'excluded') {
+ $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
+ $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1;
+ } elsif ($result eq 'fail') {
+ $pkg_tree{$prefix}{media}{$media}{failed_arch}{$arch} = 1;
+ } else {
+ plog('WARNING', "unknown state $arch.$result for $prefix");
+ }
+ } elsif ($suffix =~ /^\.(\w+)$/) {
+ my $action = $1;
+ if ($action eq 'upload') {
+ plog('DEBUG', "found already uploaded ($prefix)");
+ $pkg_tree{$prefix}{media}{$media}{uploaded} = 1;
+ }
+ }
+ }
+ }
+
+ apply_to_upload_tree($done, \&done_func);
+ apply_to_upload_tree($todo, \&todo_func);
+
+ return %pkg_tree;
+}
+
+sub read_line {
+ my $file = shift;
+
+ open my $FILE, "<$file" or die "FATAL: can't open $file for reading";
+ my $contents = <$FILE>;
+
+ $contents;
+}
+
diff --git a/ulri b/ulri
index b662232..5fb309d 100755
--- a/ulri
+++ b/ulri
@@ -35,7 +35,7 @@ use strict;
use MDK::Common qw(any cat_ if_ find);
use Iurt::Config qw(config_usage get_date config_init get_author_email check_arch check_noarch);
use Iurt::Process qw(check_pid);
-use Iurt::File qw(check_upload_tree cleanup_failed_build);
+use Iurt::File qw(cleanup_failed_build get_upload_tree_state);
use Iurt::Mail qw(sendmail);
use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput);
use File::Copy 'move';
@@ -204,8 +204,6 @@ if ($df->{per} == 100) {
($fulldate, $daydate) = get_date();
-my %pkg_tree;
-
my $todo = "$config->{queue}/todo";
my $failure = "$config->{queue}/failure";
my $done = "$config->{queue}/done";
@@ -219,78 +217,7 @@ plog('MSG', "check uploads tree");
plog('DEBUG', "input queue is $todo");
-sub todo_func {
- my ($todo, $f, $m, $s, $r) = @_;
-
- my $media = "$m/$s";
-
- if ($r =~ /(\d{14}\.(\w+)\.\w+\.\d+)_(.*\.src\.rpm)$/) {
- my ($prefix, $user, $srpm) = ($1, $2, $3);
-
- plog('DEBUG', "found srpm $srpm ($prefix)");
- $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
- $pkg_tree{$prefix}{target} = $f;
- $pkg_tree{$prefix}{user} = $user;
- push @{$pkg_tree{$prefix}{srpms}} , $srpm;
- my ($name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/;
-
- return $pkg_tree{$prefix}{srpm_name}{$name} = $srpm;
- }
-
- if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) {
- my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6);
-
- # Set path here too has we may have a lock without the src.rpm
- $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
-
- $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
- plog('DEBUG', "found lock on $host/$arch for $prefix");
-
- if ($arch =~ /noarch/) {
- plog('DEBUG', "... and $prefix is noarch");
- $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1;
- $arch =~ s/-.*//;
- }
-
- # this should be in the cache, but waiting for a cache-clean option
- $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
-
- my $time = read_line("$todo/$f/$m/$s/$r");
- $time = (split ' ', $time)[2];
- push @{$pkg_tree{$prefix}{media}{$media}{bot}}, {
- bot => $bot,
- host => $host,
- date => $date,
- pid => $pid,
- 'arch' => $arch,
- 'time' => $time
- };
- }
-
- if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) {
- my $prefix = $1;
- my @deps = map { chomp(); $_ } cat_("$todo/$f/$m/$s/$r");
- plog('DEBUG', "Adding dependency $_ ($prefix)") foreach @deps;
-
- $pkg_tree{$prefix}{deps} = \@deps;
- }
-}
-
-sub done_func {
- my ($_todo, $_f, $m, $s, $r) = @_;
-
- my $media = "$m/$s";
-
- if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.(done|fail|excluded)$/) {
- my ($prefix, $arch) = ($1, $2);
- $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
- $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
- } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
- my ($prefix, $rpm) = ($1, $2);
- plog('DEBUG', "found already built rpm $rpm ($prefix)");
- push @{$pkg_tree{$prefix}{rpms}} , $rpm;
- }
-}
+my %pkg_tree = get_upload_tree_state($config);
sub exclude_machine {
my ($config, $host) = @_;
@@ -300,15 +227,8 @@ sub exclude_machine {
}
}
-check_upload_tree($todo, \&todo_func);
-
-# getting already compiled packages
-# The cache should not be needed if the .done file are removed as the same
-# time as the src.rpm in the todo tree
-check_upload_tree($done, \&done_func);
-
# Load bot status
-
+#
foreach my $prefix (keys %pkg_tree) {
my $ent = $pkg_tree{$prefix};
foreach my $media (keys %{$ent->{media}}) {
@@ -586,9 +506,10 @@ foreach my $prefix (sort keys %pkg_tree) {
foreach my $arch (@arch_list) {
# Skip this arch if package is already building as noarch or for this arch
- #
+ # or if it should not be built on this arch
next if $pkg_tree{$prefix}{media}{$media}{arch}{noarch};
next if $pkg_tree{$prefix}{media}{$media}{arch}{$arch};
+ next if $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch};
# If all packages in a group are noarch, consider the entire group
# as noarch
@@ -603,6 +524,7 @@ foreach my $prefix (sort keys %pkg_tree) {
plog('WARN', "excluding from $arch: $excluded");
create_file("$done_dir/${prefix}_$arch.excluded",
"ulri $arch excluded");
+ $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1;
next;
}
@@ -830,15 +752,6 @@ sub create_file {
print $FILE "@contents";
}
-sub read_line {
- my $file = shift;
-
- open my $FILE, "<$file" or die "FATAL: can't open $file for reading";
- my $contents = <$FILE>;
-
- $contents;
-}
-
sub check_file_timeout {
my ($file, $time) = @_;