Add boot timeout

Boot process should timeout after selected number of seconds if no
output is generated. This allows resolving some problems with possible
boot stuck.

5 files changed, 62 insertions, 11 deletions

diff --git a/scripts/boot.py b/scripts/boot.py
index af39b30..31328a1 100644
--- a/scripts/boot.py
+++ b/scripts/boot.py
@@ -3,16 +3,28 @@ import sys
 import subprocess
 import shutil
 import importlib
+import traceback
 
 import utils
 import initialize
 from conf import conf
 from conf import sf
-from exceptions import MissingFile
+import exceptions
 import database
 
 def boot(config, to_database = True):
-	out = utils.callsubprocess('boot', conf.boot_command, conf.boot_output, True)
+	try:
+		out = utils.callsubprocess('boot', conf.boot_command, conf.boot_output, \
+				True, timeout = conf.boot_timeout)
+		result = 'nominal'
+	except exceptions.ProcessFailed as e:
+		result = 'failed'
+		out = e.output
+		traceback.print_exc()
+	except exceptions.ProcessTimeout as e:
+		result = 'timeout'
+		out = e.output
+		traceback.print_exc()
 
 	value = None
 	try:
@@ -27,4 +39,4 @@ def boot(config, to_database = True):
 			txt = ''
 			for ln in out:
 				txt += ln + '\n'
-			dtb.add_measure(txt, config.id, value)
+			dtb.add_measure(txt, result, config.id, value)
diff --git a/scripts/database.py b/scripts/database.py
index 8664393..27c6a92 100644
--- a/scripts/database.py
+++ b/scripts/database.py
@@ -99,17 +99,18 @@ class database:
 			rtn.append(Config(dt[0], hash, dt[1].split('\n')))
 		return rtn
 
-	def add_measure(self, output, conf_id, value = None):
+	def add_measure(self, output, result, conf_id, value = None):
 		"Add measurement."
 		ps = self.db.prepare("""INSERT INTO measure
-								(conf, output, value, mtime, toolgit, linuxgit, measurement)
+								(conf, output, value, mtime, toolgit,
+								linuxgit, measurement, result)
 								VALUES
-								($1, $2, $3, $4, $5, $6, $7);
+								($1, $2, $3, $4, $5, $6, $7, $8);
 								""")
 		gt = self.check_toolsgit()
 		lgt = self.check_linuxgit()
 		tm = datetime.datetime.now()
-		ps(conf_id, output, value, tm, gt, lgt, conf.measure_identifier)
+		ps(conf_id, output, value, tm, gt, lgt, conf.measure_identifier, result)
 
 	def update_measure(self, measure_id, value):
 		"Update measured value"
diff --git a/scripts/databaseinit.sql b/scripts/databaseinit.sql
index 0ca9b5a..c74640c 100644
--- a/scripts/databaseinit.sql
+++ b/scripts/databaseinit.sql
@@ -29,6 +29,7 @@ CREATE TABLE measure (
 	conf BIGINT REFERENCES configurations (id), -- Reference to configuration
 	measurement TEXT NOT NULL, -- Text identifivator of measuring tool
 	output TEXT NOT NULL, -- Output of boot
+	result TEXT NOT NULL, -- Result of boot script, if exited normally
 	value DOUBLE PRECISION DEFAULT null, -- Measured data value
 	mtime timestamp NOT NULL, -- Time and date of measurement
 	linuxgit BIGINT REFERENCES linuxgit (id), -- Reference to git version of Linux
diff --git a/scripts/exceptions.py b/scripts/exceptions.py
index 649c235..889dbe6 100644
--- a/scripts/exceptions.py
+++ b/scripts/exceptions.py
@@ -32,13 +32,21 @@ class NoApplicableConfiguration(Exception):
 		return "No applicable configuration find. All generated configurations were already applied."
 
 class ProcessFailed(Exception):
-	def __init__(self, process, returncode):
+	def __init__(self, process, returncode, output):
 		self.process = process
 		self.returncode = returncode
+		self.output = output
 	def __str__(self):
 		return "Process failed: " + str(self.process) + \
 			" with return code: " + str(self.returncode)
 
+class ProcessTimeout(Exception):
+	def __init__(self, process, output):
+		self.process = process
+		self.output = output
+	def __str__(self):
+		return "Process timeout: " + str(self.process)
+
 class DatabaseUninitialized(Exception):
 	def __str__(self):
 		return "Database seems to be uninitialized."
diff --git a/scripts/utils.py b/scripts/utils.py
index 4260cc9..f6a4720 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -3,7 +3,9 @@ import sys
 import subprocess
 import time
 import hashlib
+import signal
 import re
+from threading import Thread
 from conf import conf
 from conf import sf
 import exceptions
@@ -27,13 +29,37 @@ def build_symbol_map():
 				w = lnn.rstrip().split(sep=':')
 				smap[int(w[0])] = w[1]
 
+class __subprocess_timer__(Thread):
+	def __init__(self, sprc, timeout):
+		Thread.__init__(self, name='subprocess_timer')
+		self.sprc = sprc
+		self.last = time.time()
+		self.exitit = False
+		self.timeout = timeout
+		self.timeouted = False
+		if timeout > 0:
+			self.start()
+	def output(self):
+		self.last = time.time()
+	def exit(self):
+		self.exitit = True
+		return self.timeouted
+	def run(self):
+		while not self.exitit:
+			now = time.time()
+			if (now - self.last) >= self.timeout:
+				self.timeouted = True
+				os.kill(self.sprc.pid, signal.SIGTERM)
+				return
+			time.sleep(1)
 
 def callsubprocess(process_name, process, show_output = True,
 		return_output = False, env=os.environ, allowed_exit_codes = [0],
-		allow_all_exit_codes = False, stdin = None):
+		allow_all_exit_codes = False, stdin = None, timeout = -1):
 	sprc = subprocess.Popen(process, stdout = subprocess.PIPE,
 			stderr = subprocess.STDOUT, stdin = subprocess.PIPE, env = env)
 
+
 	try:
 		os.mkdir(os.path.join(sf(conf.log_folder), process_name))
 	except OSError:
@@ -46,21 +72,24 @@ def callsubprocess(process_name, process, show_output = True,
 		sprc.stdin.close()
 
 	rtn = []
+	timerout = __subprocess_timer__(sprc, timeout)
 	with open(os.path.join(sf(conf.log_folder),
 			process_name, time.strftime("%y-%m-%d-%H-%M-%S") + ".log"),
 			"a") as f:
 		f.write('::' + time.strftime("%y-%m-%d-%H-%M-%S-%f") + '::\n')
 		for linen in sprc.stdout:
+			timerout.output()
 			line = linen.decode(sys.getdefaultencoding())
 			f.write(line)
 			if show_output:
 				print(line, end="")
 			if return_output:
 				rtn.append(line.rstrip())
-
+	if timerout.exit():
+		raise exceptions.ProcessTimeout(process_name, rtn)
 	rtncode = sprc.wait()
 	if rtncode not in allowed_exit_codes and not allow_all_exit_codes:
-		raise exceptions.ProcessFailed(process, rtncode)
+		raise exceptions.ProcessFailed(process, rtncode, rtn)
 	return rtn
 
 def get_kernel_env():