Use a shared function to load the state of upload tree

3 files changed, 143 insertions, 184 deletions

diff --git a/emi b/emi
index 750df44..d6f1b6d 100755
--- a/emi
+++ b/emi
@@ -32,7 +32,7 @@ use File::Path qw(make_path);
 use Iurt::Config qw(config_usage config_init get_author_email get_mandatory_arch);
 use Iurt::Process qw(check_pid);
 use Iurt::Mail qw(sendmail);
-use Iurt::File qw(check_upload_tree);
+use Iurt::File qw(get_upload_tree_state);
 use Iurt::Util qw(plog_init plog);
 use Data::Dumper;
 use MDK::Common qw(cat_ if_ find touch);
@@ -118,70 +118,7 @@ my $todo = "$config->{queue}/todo";
 my $done = "$config->{queue}/done";
 my $reject = "$config->{queue}/rejected";
 
-my %pkg_tree;
-
-
-#
-# Gather data from upload tree
-#
-
-sub done_func {
-    my ($_todo, $_f, $m, $s, $r) = @_;
-
-    my $media = "$m/$s";
-
-    if ($r =~ /^(\d{14}\.\w+\.\w+\.\d+)([_.].+)$/) {
-	my ($prefix, $suffix) = ($1, $2);
-	if ($suffix =~ /^_(\w+)\.(\w+)$/) {
-	    my ($arch, $result) = ($1, $2);
-	    if ($result eq 'done') {
-		plog('DEBUG', "found .done ($prefix) for $arch");
-		$pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
-	    } elsif ($result eq 'excluded') {
-		$arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
-		plog('DEBUG', "found .excluded ($prefix) for $arch");
-		$pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1;
-	    }
-	} elsif ($suffix =~ /^\.(\w+)$/) {
-	    my $action = $1;
-	    if ($action eq 'upload') {
-		plog('DEBUG', "found already uploaded ($prefix)");
-		$pkg_tree{$prefix}{media}{$media}{uploaded} = 1;
-	    }
-	}
-    }
-}
-
-sub done_post {
-    my ($_todo, $f, $m, $s, $r) = @_;
-    my $media = "$m/$s";
-    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
-	my ($prefix, $rpm, $arch) = ($1, $2, $3);
-	$arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
-	plog('DEBUG', "found rpm $rpm ($prefix) for media $media");
-	$pkg_tree{$prefix}{target} = $f;
-	$pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
-	if ($arch eq 'src') {
-	    push @{$pkg_tree{$prefix}{media}{$media}{srpms}}, $rpm;
-	    $pkg_tree{$prefix}{media}{$media}{arch}{src} = 1;
-	}
-	push @{$pkg_tree{$prefix}{media}{$media}{rpms}} , $rpm;
-    }
-}
-
-sub todo_func {
-    my ($_todo, $_f, $_m, $_s, $r) = @_;
-
-    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
-	my ($prefix, $rpm) = ($1, $2);
-	plog('DEBUG', "found todo rpm $rpm ($prefix)");
-	push @{$pkg_tree{$prefix}{srpms}}, $rpm;
-    }
-}
-
-check_upload_tree($done, \&done_func, \&done_post);
-check_upload_tree($todo, \&todo_func);
-
+my %pkg_tree = get_upload_tree_state($config);
 
 #
 # Decide what should be uploaded
diff --git a/lib/Iurt/File.pm b/lib/Iurt/File.pm
index b91ec2c..33be6a1 100644
--- a/lib/Iurt/File.pm
+++ b/lib/Iurt/File.pm
@@ -5,55 +5,45 @@ use Iurt::Config qw(get_mandatory_arch);
 use File::Copy 'move';
 use File::Path 'make_path';
 use Iurt::Util qw(plog);
-use MDK::Common qw(find member partition);
+use MDK::Common qw(cat_ find member partition);
 use strict;
 
 our @EXPORT = qw(
-    check_upload_tree 
+    get_upload_tree_state
     cleanup_failed_build
 );
 
-sub check_upload_tree {
-    my ($todo, $func, $o_post) = @_;
+sub apply_to_upload_tree {
+    my ($tree_root, $func) = @_;
 
     # Squash double slashes for cosmetics
-    $todo =~ s!/+!/!g;
+    $tree_root =~ s!/+!/!g;
 
-    opendir(my $dir, $todo);
-    plog('INFO', "check dir: $todo");
+    opendir(my $dir, $tree_root);
+    plog('INFO', "check dir: $tree_root");
 
     foreach my $f (readdir $dir) {
 	$f =~ /^\.{1,2}$/ and next;
-	if (-d "$todo/$f") {
-	    plog('DEBUG', "checking target $todo/$f");
-	    opendir my $target_dir, "$todo/$f";
+	if (-d "$tree_root/$f") {
+	    plog('DEBUG', "checking target $tree_root/$f");
+	    opendir my $target_dir, "$tree_root/$f";
 
 	    foreach my $m (readdir $target_dir) {
 		$m =~ /^\.{1,2}$/ and next;
-		if (-d "$todo/$f/$m") {
-		    plog('DEBUG', "checking media $todo/$f/$m");
-		    opendir my $media_dir, "$todo/$f/$m";
+		if (-d "$tree_root/$f/$m") {
+		    plog('DEBUG', "checking media $tree_root/$f/$m");
+		    opendir my $media_dir, "$tree_root/$f/$m";
 
 		    foreach my $s (readdir $media_dir) {
 			$s =~ /^\.{1,2}$/ and next;
-			if (-d "$todo/$f/$m/$s") {
+			if (-d "$tree_root/$f/$m/$s") {
 			    if ($func) {
-				opendir my $submedia_dir, "$todo/$f/$m/$s";
+				opendir my $submedia_dir, "$tree_root/$f/$m/$s";
 				foreach my $r (readdir $submedia_dir) {
 				    $r =~ /^\.{1,2}$/ and next;
-				    $func->($todo, $f, $m, $s, $r);
+				    $func->($tree_root, $f, $m, $s, $r);
 				}
 			    }
-			    # cleaning
-			    if ($o_post) {
-				opendir my $submedia_dir, "$todo/$f/$m/$s";
-				foreach my $r (readdir $submedia_dir) {
-				    $r =~ /^\.{1,2}$/ and next;
-				    $o_post->($todo, $f, $m, $s, $r);
-				}
-			    }
-			} else {
-			    # may need to check also here for old target
 			}
 		    }
 		}
@@ -113,3 +103,122 @@ sub cleanup_failed_build {
     }
 }
 
+sub get_upload_tree_state {
+    our ($config) = @_;
+
+    our %pkg_tree;
+    my $todo = "$config->{queue}/todo";
+    my $done = "$config->{queue}/done";
+
+    sub todo_func {
+	my ($todo, $f, $m, $s, $r) = @_;
+
+	my $media = "$m/$s";
+
+	if ($r =~ /(\d{14}\.(\w+)\.\w+\.\d+)_(.*\.src\.rpm)$/) {
+	    my ($prefix, $user, $srpm) = ($1, $2, $3);
+
+	    plog('DEBUG', "found srpm $srpm ($prefix)");
+	    $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
+	    $pkg_tree{$prefix}{target} = $f;
+	    $pkg_tree{$prefix}{user} = $user;
+	    push @{$pkg_tree{$prefix}{srpms}} , $srpm;
+	    my ($name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/;
+
+	    $pkg_tree{$prefix}{srpm_name}{$name} = $srpm;
+	}
+
+	if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) {
+	    my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6);
+
+	    # Set path here too has we may have a lock without the src.rpm
+	    $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
+
+	    $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
+	    plog('DEBUG', "found lock on $host/$arch for $prefix");
+
+	    if ($arch =~ /noarch/) {
+		plog('DEBUG', "... and $prefix is noarch");
+		$pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1;
+		$arch =~ s/-.*//;
+	    }
+
+	    $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
+
+	    my $time = read_line("$todo/$f/$m/$s/$r");
+	    $time = (split ' ', $time)[2];
+	    push @{$pkg_tree{$prefix}{media}{$media}{bot}}, {
+		bot => $bot,
+		host => $host,
+		date => $date,
+		pid => $pid,
+		'arch' => $arch,
+		'time' => $time
+	    };
+	}
+
+	if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) {
+	    my $prefix = $1;
+	    my @deps = map { chomp(); $_ } cat_("$todo/$f/$m/$s/$r");
+	    plog('DEBUG', "Adding dependency $_ ($prefix)") foreach @deps;
+
+	    $pkg_tree{$prefix}{deps} = \@deps;
+	}
+    }
+
+    sub done_func {
+	my ($_todo, $f, $m, $s, $r) = @_;
+
+	my $media = "$m/$s";
+
+	if ($r =~ /^(\d{14}\.\w+\.\w+\.\d+)([_.].+)$/) {
+	    my ($prefix, $suffix) = ($1, $2);
+	    $pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
+	    if ($suffix =~ /^_(.*\.([^.]+)\.rpm)$/) {
+		my ($rpm, $arch) = ($1, $2);
+		$arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
+		plog('DEBUG', "found already built rpm $rpm ($prefix) for media $media");
+		$pkg_tree{$prefix}{target} = $f;
+		if ($arch eq 'src') {
+		    $pkg_tree{$prefix}{media}{$media}{arch}{src} = 1;
+		}
+		push @{$pkg_tree{$prefix}{media}{$media}{rpms}} , $rpm;
+		push @{$pkg_tree{$prefix}{rpms}} , $rpm;
+	    } elsif ($suffix =~ /^_(\w+)\.(\w+)$/) {
+		my ($arch, $result) = ($1, $2);
+		plog('DEBUG', "found .$result ($prefix) for $arch");
+		if ($result eq 'done') {
+		    $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
+		} elsif ($result eq 'excluded') {
+		    $arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
+		    $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1;
+		} elsif ($result eq 'fail') {
+		    $pkg_tree{$prefix}{media}{$media}{failed_arch}{$arch} = 1;
+		} else {
+		    plog('WARNING', "unknown state $arch.$result for $prefix");
+		}
+	    } elsif ($suffix =~ /^\.(\w+)$/) {
+		my $action = $1;
+		if ($action eq 'upload') {
+		    plog('DEBUG', "found already uploaded ($prefix)");
+		    $pkg_tree{$prefix}{media}{$media}{uploaded} = 1;
+		}
+	    }
+	}
+    }
+
+    apply_to_upload_tree($done, \&done_func);
+    apply_to_upload_tree($todo, \&todo_func);
+
+    return %pkg_tree;
+}
+
+sub read_line {
+    my $file = shift;
+
+    open my $FILE, "<$file" or die "FATAL: can't open $file for reading";
+    my $contents = <$FILE>;
+
+    $contents;
+}
+
diff --git a/ulri b/ulri
index b662232..5fb309d 100755
--- a/ulri
+++ b/ulri
@@ -35,7 +35,7 @@ use strict;
 use MDK::Common qw(any cat_ if_ find);
 use Iurt::Config qw(config_usage get_date config_init get_author_email check_arch check_noarch);
 use Iurt::Process qw(check_pid);
-use Iurt::File qw(check_upload_tree cleanup_failed_build);
+use Iurt::File qw(cleanup_failed_build get_upload_tree_state);
 use Iurt::Mail qw(sendmail);
 use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput);
 use File::Copy 'move';
@@ -204,8 +204,6 @@ if ($df->{per} == 100) {
 
 ($fulldate, $daydate) = get_date();
 
-my %pkg_tree;
-
 my $todo = "$config->{queue}/todo";
 my $failure = "$config->{queue}/failure";
 my $done = "$config->{queue}/done";
@@ -219,78 +217,7 @@ plog('MSG', "check uploads tree");
 
 plog('DEBUG', "input queue is $todo");
 
-sub todo_func {
-    my ($todo, $f, $m, $s, $r) = @_;
-
-    my $media = "$m/$s";
-
-    if ($r =~ /(\d{14}\.(\w+)\.\w+\.\d+)_(.*\.src\.rpm)$/) {
-	my ($prefix, $user, $srpm) = ($1, $2, $3);
-
-	plog('DEBUG', "found srpm $srpm ($prefix)");
-	$pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
-	$pkg_tree{$prefix}{target} = $f;
-	$pkg_tree{$prefix}{user} = $user;
-	push @{$pkg_tree{$prefix}{srpms}} , $srpm;
-	my ($name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm$/;
-
-	return $pkg_tree{$prefix}{srpm_name}{$name} = $srpm;
-    }
-
-    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_([\w-]+)\.(\w+)\.(\w+)\.(\d{14})\.(\d+)\.lock$/) {
-	my ($prefix, $arch, $bot, $host, $date, $pid) = ($1, $2, $3, $4, $5, $6);
-
-	# Set path here too has we may have a lock without the src.rpm
-	$pkg_tree{$prefix}{media}{$media}{path} = "/$f/$m/$s";
-
-	$arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
-	plog('DEBUG', "found lock on $host/$arch for $prefix");
-
-	if ($arch =~ /noarch/) {
-	    plog('DEBUG', "... and $prefix is noarch");
-	    $pkg_tree{$prefix}{media}{$media}{arch}{noarch} = 1;
-	    $arch =~ s/-.*//;
-	}
-
-	# this should be in the cache, but waiting for a cache-clean option
-	$pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
-
-	my $time = read_line("$todo/$f/$m/$s/$r");
-	$time = (split ' ', $time)[2];
-	push @{$pkg_tree{$prefix}{media}{$media}{bot}}, {
-	    bot => $bot,
-	    host => $host,
-	    date => $date,
-	    pid => $pid,
-	    'arch' => $arch,
-	    'time' => $time
-	};
-    }
-
-    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_.*\.deps$/) {
-	my $prefix = $1;
-	my @deps = map { chomp(); $_ } cat_("$todo/$f/$m/$s/$r");
-	plog('DEBUG', "Adding dependency $_ ($prefix)") foreach @deps;
-
-	$pkg_tree{$prefix}{deps} = \@deps;
-    }
-}
-
-sub done_func {
-    my ($_todo, $_f, $m, $s, $r) = @_;
-
-    my $media = "$m/$s";
-
-    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.(done|fail|excluded)$/) {
-	my ($prefix, $arch) = ($1, $2);
-	$arch = $config->{arch_translation}{$arch} if $config->{arch_translation}{$arch};
-	$pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 1;
-    } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
-	my ($prefix, $rpm) = ($1, $2);
-	plog('DEBUG', "found already built rpm $rpm ($prefix)");
-	push @{$pkg_tree{$prefix}{rpms}} , $rpm;
-    }
-}
+my %pkg_tree = get_upload_tree_state($config);
 
 sub exclude_machine {
     my ($config, $host) = @_;
@@ -300,15 +227,8 @@ sub exclude_machine {
     }
 }
 
-check_upload_tree($todo, \&todo_func);
-
-# getting already compiled packages
-# The cache should not be needed if the .done file are removed as the same
-# time as the src.rpm in the todo tree
-check_upload_tree($done, \&done_func);
-
 # Load bot status
-
+#
 foreach my $prefix (keys %pkg_tree) {
     my $ent = $pkg_tree{$prefix};
     foreach my $media (keys %{$ent->{media}}) {
@@ -586,9 +506,10 @@ foreach my $prefix (sort keys %pkg_tree) {
 	foreach my $arch (@arch_list) {
 	    
 	    # Skip this arch if package is already building as noarch or for this arch
-	    #
+	    #  or if it should not be built on this arch
 	    next if $pkg_tree{$prefix}{media}{$media}{arch}{noarch};
 	    next if $pkg_tree{$prefix}{media}{$media}{arch}{$arch};
+	    next if $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch};
 	    
 	    # If all packages in a group are noarch, consider the entire group
 	    # as noarch
@@ -603,6 +524,7 @@ foreach my $prefix (sort keys %pkg_tree) {
 		plog('WARN', "excluding from $arch: $excluded");
 		create_file("$done_dir/${prefix}_$arch.excluded",
 			    "ulri $arch excluded");
+		$pkg_tree{$prefix}{media}{$media}{excluded_arch}{$arch} = 1;
 		next;
 	    }
 	    
@@ -830,15 +752,6 @@ sub create_file {
     print $FILE "@contents";
 }
 
-sub read_line {
-    my $file = shift;
-
-    open my $FILE, "<$file" or die "FATAL: can't open $file for reading";
-    my $contents = <$FILE>;
-
-    $contents;
-}
-
 sub check_file_timeout {
     my ($file, $time) = @_;