From a52f4d1693ae1114fc188a70db8056780cbf4cf1 Mon Sep 17 00:00:00 2001
From: Pascal Terjan <pterjan@gmail.com>
Date: Mon, 3 Oct 2016 22:58:07 +0100
Subject: Do not give up on a build when failing to check its status

---
 NEWS |   1 +
 ulri | 136 +++++++++++++++++++++++++++++++++----------------------------------
 2 files changed, 68 insertions(+), 69 deletions(-)

diff --git a/NEWS b/NEWS
index 0f206a5..4dbb12f 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,7 @@
   o share the code loading the state from the file tree
 - add first test
 - improve arch compatibility checking
+- do not give up on a build when failing to check its status
 
 0.6.25
 - drop arcfour cipher usage (not enabled by default in latest openssh packages)
diff --git a/ulri b/ulri
index 1cb9bf8..68b4415 100755
--- a/ulri
+++ b/ulri
@@ -277,84 +277,87 @@ foreach my $prefix (keys %pkg_tree) {
 
 	    plog('INFO', "check status: $host/$arch ($bot [$pid])");
 	    my $status = sout($remote, "cat $status_file");
+	    if (!$status) {
+		plog('WARN', "failed to get status for $host/$arch");
+	        next bot;
+	    }
+
 	    my $success;
 	    my $fail;
 	    my $later;
 
 	    # Check if the build bot finished on the other side
 	    #
-	    if ($status) {
-		plog('INFO', "check result: $host/$arch ($bot [$pid])");
-		foreach my $res (split "\n", $status) {
-		    my ($p, $r) = $res =~ /(.*):\s+(.*)/;
-		    plog('DEBUG', $res);
-		    if ($r eq 'install_deps_failure') {
-			plog('FAIL', "install deps failure, rebuild later: $p");
-			$later{$prefix} = 1;
-			$later = 1;
-		    }
-		    if ($r ne 'ok') {
-			plog('FAIL', "$r: $p");
-			$fail = 1;
-		    }
+	    plog('INFO', "check result: $host/$arch ($bot [$pid])");
+	    foreach my $res (split "\n", $status) {
+		my ($p, $r) = $res =~ /(.*):\s+(.*)/;
+		plog('DEBUG', $res);
+		if ($r eq 'install_deps_failure') {
+		    plog('FAIL', "install deps failure, rebuild later: $p");
+		    $later{$prefix} = 1;
+		    $later = 1;
 		}
+		if ($r ne 'ok') {
+		    plog('FAIL', "$r: $p");
+		    $fail = 1;
+		}
+	    }
+
+	    if (!$fail) {
+		my @list = split "\n", sout($remote, "ls $prefix_dir");
+		my $error;
+		my $done;
+
+		my $arch_check = join '|', $arch, if_($untranslated_arch{$arch}, @{$untranslated_arch{$arch}});
+		plog('MSG', "checking for $arch_check arch");
+		foreach my $result (@list) {
+		    $result =~ /\.(src|$arch_check|noarch)\.rpm$/ or next;
+
+		    # do not copy the initial src package
+		    $result =~ /^$prefix/ and next;
 
-		if (!$fail) {
-		    my @list = split "\n", sout($remote, "ls $prefix_dir");
-		    my $error;
-		    my $done;
-
-		    my $arch_check = join '|', $arch, if_($untranslated_arch{$arch}, @{$untranslated_arch{$arch}});
-		    plog('MSG', "checking for $arch_check arch");
-		    foreach my $result (@list) {
-			$result =~ /\.(src|$arch_check|noarch)\.rpm$/ or next;
-
-			# do not copy the initial src package
-			$result =~ /^$prefix/ and next;
-
-			my $result_file = "$done_dir/${prefix}_$result";
-
-			plog('OK', "build ok: $result");
-			if ($result =~ /\.$arch_check\.rpm$/) {
-			    $done = 1;
-			}
-
-			plog('DEBUG', "copy files to done");
-			make_path($done_dir);
-			if (sget($remote, "$prefix_dir/$result",
-				 "$result_file.new")) {
-			    plog('ERROR', "copying $result from $host failed ($!)");
-			    $error = 1;
-			    last;
-			}
-		       	if (!move("$result_file.new", $result_file)) {
-			    $error = 1;
-			    last;
-			}
-			# Add the package to the list of built ones, in case we fail another arch and need to cleanup
-			push @{$ent->{rpms}}, $result_file;
+		    my $result_file = "$done_dir/${prefix}_$result";
+
+		    plog('OK', "build ok: $result");
+		    if ($result =~ /\.$arch_check\.rpm$/) {
+			$done = 1;
 		    }
-		    next if $error;
 
-		    if ($done) {
-			create_file("$done_dir/${prefix}_$arch.done", "$bot $host");
-			$success = 1;
+		    plog('DEBUG', "copy files to done");
+		    make_path($done_dir);
+		    if (sget($remote, "$prefix_dir/$result",
+			     "$result_file.new")) {
+			plog('ERROR', "copying $result from $host failed ($!)");
+			$error = 1;
+			last;
+		    }
+		    if (!move("$result_file.new", $result_file)) {
+			$error = 1;
+			last;
 		    }
+		    # Add the package to the list of built ones, in case we fail another arch and need to cleanup
+		    push @{$ent->{rpms}}, $result_file;
+		}
+		next if $error;
+
+		if ($done) {
+		    create_file("$done_dir/${prefix}_$arch.done", "$bot $host");
+		    $success = 1;
+		}
 
-		    if ($success) {
-			# Fetch build log and clean remote machine
-			make_path("$done_dir/$prefix");
-			sget($remote, "$prefix_dir/log/*", "$done_dir/$prefix");
-			ssh($remote, "rm -rf $prefix_dir");
-			if (-d "$fail_dir/$prefix") {
-			    # Other arch had failed
-			    cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $arch, $config);
-			} else {
-			    $something_finished = 1;
-			}
+		if ($success) {
+		    # Fetch build log and clean remote machine
+		    make_path("$done_dir/$prefix");
+		    sget($remote, "$prefix_dir/log/*", "$done_dir/$prefix");
+		    ssh($remote, "rm -rf $prefix_dir");
+		    if (-d "$fail_dir/$prefix") {
+			# Other arch had failed
+			cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $arch, $config);
+		    } else {
+			$something_finished = 1;
 		    }
 		}
-	    } # if ($status)
+	    }
 
 	    #
 	    # Handle build failure
@@ -393,11 +396,6 @@ foreach my $prefix (keys %pkg_tree) {
 		next bot;
 	    }
 
-	    if (!$status) {
-	    plog('ERROR', "build bot died on $host, reschedule compilation");
-	    next bot;
-	    }
-
 	    plog('INFO', "Failure reason: $success || $later || $fail || $timeout || $zombie || $ended");
 
 	    plog('FAIL', "build failed");
-- 
cgit v1.2.1