Increase robustness when scp fails0.8.0

Do not remove lock file and mark build as done until we copied all
the files.

1 files changed, 15 insertions, 32 deletions

diff --git a/ulri b/ulri
index bbce305..93538f3 100755
--- a/ulri
+++ b/ulri
@@ -25,10 +25,10 @@ use Iurt::Config qw(config_usage get_date config_init get_author_email get_targe
 use Iurt::File qw(create_file);
 use Iurt::Mail qw(sendmail);
 use Iurt::Process qw(check_pid);
-use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state load_lock_file_data remove_bot_from_package);
+use Iurt::Queue qw(check_if_mandatory_arch_failed cleanup_failed_build get_upload_tree_state load_lock_file_data record_bot_complete);
 use Iurt::RPM qw(check_arch check_noarch);
 use Iurt::Util qw(plog_init plog ssh_setup ssh sout sget sput);
-use Iurt::Ulri qw(build_package warn_about_failure);
+use Iurt::Ulri qw(build_package fetch_logs_and_cleanup warn_about_failure);
 use File::Copy 'move';
 use File::Path 'make_path';
 use File::Temp 'mktemp';
@@ -305,19 +305,10 @@ foreach my $prefix (keys %pkg_tree) {
 		} else {
 		    plog('FAIL', "$bot died on $host/$arch (status $proc_state), removing lock");
 	        }
-	        $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0;
-	    }
-
-	    # Either we are done or we should kill the build
-
-	    plog('INFO', "delete lock file for $prefix");
-	    unlink $lock_file;
 
-	    $run{bot}{$host}{$bot} = 0;
-	    remove_bot_from_package($ent, $media, $host, $pid);
+		fetch_and_delete_logs($remote, "$prefix_dir/log", "$fail_dir/$prefix");
+		record_bot_complete($run, $bot, $lock_file, $prefix, $ent, $media, $host, $pid);
 
-	    if (!$status) {
-		# TODO: fetch/clean the logs
 		next bot;
 	    }
 
@@ -335,8 +326,7 @@ foreach my $prefix (keys %pkg_tree) {
 		    plog('FAIL', "install deps failure, rebuild later: $p");
 		    $later{$prefix} = 1;
 		    $later = 1;
-		    # TODO: fetch/clean the logs
-		}
+	        }
 		if ($r ne 'ok') {
 		    plog('FAIL', "$r: $p");
 		    $fail = 1;
@@ -376,38 +366,31 @@ foreach my $prefix (keys %pkg_tree) {
 		    # Add the package to the list of built ones, in case we fail another arch and need to cleanup
 		    push @{$ent->{rpms}}, $result_file;
 		}
-		next if $error;
+		next bot if $error;
 
 		if (check_if_mandatory_arch_failed($media, $ent, $config)) {
 		    # Discard this arch as another mandatory one failed
 		    cleanup_failed_build($todo_dir, $done_dir, $fail_dir, $prefix, $ent, $media, $arch, $config);
+		    ssh($remote, "rm -rf $prefix_dir");
 		} else {
 		    create_file("$done_dir/${prefix}_$arch.done", "$bot $host");
 		    $pkg_tree{$prefix}{media}{$media}{done_arch}{$arch} = 1;
-		    make_path("$done_dir/$prefix");
-		    sget($remote, "$prefix_dir/log/*", "$done_dir/$prefix");
+		    fetch_logs_and_cleanup($remote, $prefix_dir, "$done_dir/$prefix");
 		    $something_finished = 1;
 		}
-		# Either we already fetched the success logs, or don't care
-		# as this success was discarded due to another failure.
-		ssh($remote, "rm -rf $prefix_dir");
-		next bot;
 	    }
 
-	    make_path($fail_dir);
+	    record_bot_complete($run, $bot, $lock_file, $prefix, $ent, $media, $host, $pid);
+
+	    # In case of success we have now fetched packages and logs and cleaned up the remote machine
+	    next bot if $done;
 
 	    unless ($pkg_tree{$prefix}{media}{$media}{cancelled_arch}{$arch}) {
-		mkdir("$fail_dir/$prefix");
-		if (sget($remote, "$prefix_dir/*", "$fail_dir/$prefix")) {
-		    plog('ERROR', "copying from $host:$prefix_dir/ " .
-		         "to $fail_dir/ failed ($!)");
-		    $pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0;
-	        }
+		make_path($fail_dir);
+		fetch_logs_and_cleanup($remote, $prefix_dir, "$fail_dir/$prefix");
+		$pkg_tree{$prefix}{media}{$media}{arch}{$arch} = 0;
 	    }
 
-	    # clean the log on the compilation machine
-	    ssh($remote, "rm -rf $prefix_dir");
-
 	    # We got the logs but want to retry so don't record a failure
 	    next bot if $later;