Kill build if nothing is printed for 5 minutes and system is idle

3 files changed, 22 insertions, 2 deletions

diff --git a/NEWS b/NEWS
index 28266a5..042119d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+- kill build if nothing is printed for 5 minutes and
+  system is idle
+
 0.6.13
 - use rpmbuild/ rather than rpm/
   (need invalidating chroots)
diff --git a/iurt b/iurt
index afe3594..fb566e4 100755
--- a/iurt
+++ b/iurt
@@ -445,6 +445,12 @@ my %config_usage = (
 	    default => 18000,
 	},
     },
+    build_stalled_timeout => {
+	desc => 'Maximum build time after which the build process is terminated if it seems stalled',
+	default => {
+	    default => 300,
+	},
+    },
     cache_home => {
 	desc => 'Where to store the cache files',
 	default => "$HOME/.bugs"
@@ -948,6 +954,7 @@ retry:
 		logname => "build", 
 		hash => "build_$srpm", 
 		timeout => $config->{build_timeout}{$srpm_name} || $config->{build_timeout}{default},
+		stalled_timeout => $config->{build_stalled_timeout}{$srpm_name} || $config->{build_stalled_timeout}{default},
 		srpm => $srpm,
 		debug_mail => $run{debug},
 		cc => $cc, 
diff --git a/lib/Iurt/Process.pm b/lib/Iurt/Process.pm
index 3ad858d..d6fadc2 100644
--- a/lib/Iurt/Process.pm
+++ b/lib/Iurt/Process.pm
@@ -8,6 +8,7 @@ use Iurt::Mail qw(sendmail);
 use Iurt::Config qw(dump_cache_par);
 use Iurt::Util qw(plog);
 use POSIX ":sys_wait_h";
+use Sys::Load qw(getload);
 
 our @EXPORT = qw(
     kill_for_good
@@ -102,14 +103,23 @@ sub fork_to_monitor {
 	    my (@stat) = stat $logfile;
 	    if ($stat[7] > $size_limit) {
 		# FIXME: we left runaway processes (eg: urpmi)
-		plog('NONE', "ERROR: killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})");
+		plog('ERROR', "Killing current command because of log size exceeding limit ($stat[7] > $config->{log_size_limit})");
 		kill 14, "-$parent_pid";
 		exit();
 	    }
+	    if ($stat[9] + $opt{stalled_timeout} < time()){
+		# If nothing was written to the logfile for more than stalled_timeout, check if the system seems busy
+		if ((getload())[1] < 0.5) {
+		    plog('ERROR', "Killing current command because it seems blocked");
+		    kill 14, "-$parent_pid";
+		    exit();
+		}
+	    }
+
 	    my $df = df $opt{log};
 	    if ($df->{per} >= 99) {
 		# FIXME: we left runaway processes (eg: urpmi)
-		plog('NONE', "ERROR: killing current command because running out of disk space at $opt{log} (only $df->{bavail}KB left)");
+		plog('ERROR', "Killing current command because running out of disk space at $opt{log} (only $df->{bavail}KB left)");
 		kill 14, "-$parent_pid";
 		exit();
 	    }