From a611965667c44f5db923a48d26e824d094e1a664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Mon, 24 Aug 2015 14:39:16 +0200 Subject: Add boot timeout Boot process should timeout after selected number of seconds if no output is generated. This allows resolving some problems with possible boot stuck. --- conf.py | 3 +++ scripts/boot.py | 18 +++++++++++++++--- scripts/database.py | 9 +++++---- scripts/databaseinit.sql | 1 + scripts/exceptions.py | 10 +++++++++- scripts/utils.py | 35 ++++++++++++++++++++++++++++++++--- targets/ryuglab/boot/boot | 3 ++- 7 files changed, 67 insertions(+), 12 deletions(-) diff --git a/conf.py b/conf.py index 9efc975..b8204cb 100644 --- a/conf.py +++ b/conf.py @@ -18,6 +18,9 @@ build_command = ['make'] # boot_command # Command executed for booting. Output of this command is saved to output folder. boot_command = ['echo', 'bootit'] +# boot_timeout +# Set timeout of boot process if no output is generated for selected seconds +boot_timeout = 120 # parse_command # Command to parse double value from boot output diff --git a/scripts/boot.py b/scripts/boot.py index af39b30..31328a1 100644 --- a/scripts/boot.py +++ b/scripts/boot.py @@ -3,16 +3,28 @@ import sys import subprocess import shutil import importlib +import traceback import utils import initialize from conf import conf from conf import sf -from exceptions import MissingFile +import exceptions import database def boot(config, to_database = True): - out = utils.callsubprocess('boot', conf.boot_command, conf.boot_output, True) + try: + out = utils.callsubprocess('boot', conf.boot_command, conf.boot_output, \ + True, timeout = conf.boot_timeout) + result = 'nominal' + except exceptions.ProcessFailed as e: + result = 'failed' + out = e.output + traceback.print_exc() + except exceptions.ProcessTimeout as e: + result = 'timeout' + out = e.output + traceback.print_exc() value = None try: @@ -27,4 +39,4 @@ def boot(config, to_database = True): txt = '' for ln in out: txt += ln + '\n' - dtb.add_measure(txt, config.id, value) + dtb.add_measure(txt, result, config.id, value) diff --git a/scripts/database.py b/scripts/database.py index 8664393..27c6a92 100644 --- a/scripts/database.py +++ b/scripts/database.py @@ -99,17 +99,18 @@ class database: rtn.append(Config(dt[0], hash, dt[1].split('\n'))) return rtn - def add_measure(self, output, conf_id, value = None): + def add_measure(self, output, result, conf_id, value = None): "Add measurement." ps = self.db.prepare("""INSERT INTO measure - (conf, output, value, mtime, toolgit, linuxgit, measurement) + (conf, output, value, mtime, toolgit, + linuxgit, measurement, result) VALUES - ($1, $2, $3, $4, $5, $6, $7); + ($1, $2, $3, $4, $5, $6, $7, $8); """) gt = self.check_toolsgit() lgt = self.check_linuxgit() tm = datetime.datetime.now() - ps(conf_id, output, value, tm, gt, lgt, conf.measure_identifier) + ps(conf_id, output, value, tm, gt, lgt, conf.measure_identifier, result) def update_measure(self, measure_id, value): "Update measured value" diff --git a/scripts/databaseinit.sql b/scripts/databaseinit.sql index 0ca9b5a..c74640c 100644 --- a/scripts/databaseinit.sql +++ b/scripts/databaseinit.sql @@ -29,6 +29,7 @@ CREATE TABLE measure ( conf BIGINT REFERENCES configurations (id), -- Reference to configuration measurement TEXT NOT NULL, -- Text identifivator of measuring tool output TEXT NOT NULL, -- Output of boot + result TEXT NOT NULL, -- Result of boot script, if exited normally value DOUBLE PRECISION DEFAULT null, -- Measured data value mtime timestamp NOT NULL, -- Time and date of measurement linuxgit BIGINT REFERENCES linuxgit (id), -- Reference to git version of Linux diff --git a/scripts/exceptions.py b/scripts/exceptions.py index 649c235..889dbe6 100644 --- a/scripts/exceptions.py +++ b/scripts/exceptions.py @@ -32,13 +32,21 @@ class NoApplicableConfiguration(Exception): return "No applicable configuration find. All generated configurations were already applied." class ProcessFailed(Exception): - def __init__(self, process, returncode): + def __init__(self, process, returncode, output): self.process = process self.returncode = returncode + self.output = output def __str__(self): return "Process failed: " + str(self.process) + \ " with return code: " + str(self.returncode) +class ProcessTimeout(Exception): + def __init__(self, process, output): + self.process = process + self.output = output + def __str__(self): + return "Process timeout: " + str(self.process) + class DatabaseUninitialized(Exception): def __str__(self): return "Database seems to be uninitialized." diff --git a/scripts/utils.py b/scripts/utils.py index 4260cc9..f6a4720 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -3,7 +3,9 @@ import sys import subprocess import time import hashlib +import signal import re +from threading import Thread from conf import conf from conf import sf import exceptions @@ -27,13 +29,37 @@ def build_symbol_map(): w = lnn.rstrip().split(sep=':') smap[int(w[0])] = w[1] +class __subprocess_timer__(Thread): + def __init__(self, sprc, timeout): + Thread.__init__(self, name='subprocess_timer') + self.sprc = sprc + self.last = time.time() + self.exitit = False + self.timeout = timeout + self.timeouted = False + if timeout > 0: + self.start() + def output(self): + self.last = time.time() + def exit(self): + self.exitit = True + return self.timeouted + def run(self): + while not self.exitit: + now = time.time() + if (now - self.last) >= self.timeout: + self.timeouted = True + os.kill(self.sprc.pid, signal.SIGTERM) + return + time.sleep(1) def callsubprocess(process_name, process, show_output = True, return_output = False, env=os.environ, allowed_exit_codes = [0], - allow_all_exit_codes = False, stdin = None): + allow_all_exit_codes = False, stdin = None, timeout = -1): sprc = subprocess.Popen(process, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, stdin = subprocess.PIPE, env = env) + try: os.mkdir(os.path.join(sf(conf.log_folder), process_name)) except OSError: @@ -46,21 +72,24 @@ def callsubprocess(process_name, process, show_output = True, sprc.stdin.close() rtn = [] + timerout = __subprocess_timer__(sprc, timeout) with open(os.path.join(sf(conf.log_folder), process_name, time.strftime("%y-%m-%d-%H-%M-%S") + ".log"), "a") as f: f.write('::' + time.strftime("%y-%m-%d-%H-%M-%S-%f") + '::\n') for linen in sprc.stdout: + timerout.output() line = linen.decode(sys.getdefaultencoding()) f.write(line) if show_output: print(line, end="") if return_output: rtn.append(line.rstrip()) - + if timerout.exit(): + raise exceptions.ProcessTimeout(process_name, rtn) rtncode = sprc.wait() if rtncode not in allowed_exit_codes and not allow_all_exit_codes: - raise exceptions.ProcessFailed(process, rtncode) + raise exceptions.ProcessFailed(process, rtncode, rtn) return rtn def get_kernel_env(): diff --git a/targets/ryuglab/boot/boot b/targets/ryuglab/boot/boot index 331d9dc..3f88f6e 100755 --- a/targets/ryuglab/boot/boot +++ b/targets/ryuglab/boot/boot @@ -9,4 +9,5 @@ cd `dirname $0` ln -sf ../../../jobfiles/linuxImage uImage ln -sf ../../../tests/cyclictest/root/images/rootfs.cpio.uboot rootfs.cpio.uboot -novaboot nbscripts --exiton="NOVABOOT EXIT LINUX-CONF-PERF" +novaboot nbscripts --exiton="NOVABOOT EXIT LINUX-CONF-PERF" --exiton-timeout=120 \ + --exiton="Kernel panic" -- cgit v1.2.3