diff options
| -rw-r--r-- | conf.py | 25 | ||||
| -rw-r--r-- | scripts/boot.py | 21 | ||||
| -rw-r--r-- | scripts/configurations.py | 67 | ||||
| -rw-r--r-- | scripts/exceptions.py | 10 | ||||
| -rwxr-xr-x | scripts/initialize.py | 45 | ||||
| -rwxr-xr-x | scripts/loop.py | 190 | ||||
| -rwxr-xr-x | scripts/test.py | 16 | ||||
| -rw-r--r-- | scripts/utils.py | 75 |
8 files changed, 196 insertions, 253 deletions
@@ -25,7 +25,11 @@ novaboot_args = ['--qemu=qemu-system-x86_64'] nbscript = 'scripts/nbscript' # boot_command # Command executed for booting. Output of this command is saved to output folder. -boot_command = ['scripts/novaboot/novaboot', nbscript] + novaboot_args +boot_command = ['echo', 'bootit'] + +# parse_command +# Command to parse double value from boot output +parse_command = ['echo', '0'] # picosat_args # Additional arguments passed to PicoSAT. @@ -47,6 +51,13 @@ db_host = 'localhost' # Port of PotgreSQL database server db_port = 5432 +# multithread +# Define if measurement and kernel build should be executed in parallel. +multithread = False +# multithread_buffer +# Defines maximal number of buffered configurations before generating is suspended. +multithread_buffer = 32 + # git_describe_cmd # Command used for getting tools version and status from git git_describe_cmd = ['git', 'describe', '--always', '--tags', '--dirty'] @@ -63,9 +74,9 @@ picosat_output = False kernel_config_output = True kernel_make_output = True boot_output = True +parse_output = False ## Configs for debugging -step_by_step = False # Executes only single step and exits. single_loop = False # Executes only one loop and exits. only_config = False # Executes only to configuration phase. Building and booting phases are skipped. ignore_misconfig = False # Ignore if configuration wasn't applied correctly. @@ -87,13 +98,14 @@ buildroot_initram = 'buildroot/output/images/rootfs.cpio.gz' build_folder = 'jobfiles/' jobfolder_linux_image = build_folder + 'linuxImage' -phase_file = build_folder + 'phase' symbol_map_file = build_folder + 'symbol_map' # Also defined in parse_kconfig rules_file = build_folder + 'rules' # Also defined in parse_kconfig variable_count_file = build_folder + 'variable_count' # Also defined in parse_kconfig -required_file = build_folder + 'required' +fixed_file = build_folder + 'fixed' +measure_file = build_folder + 'measure' +dot_measure_file = build_folder + 'dot_measure' dot_config_back_file = build_folder + 'dot_config_back' -iteration_file = build_folder + 'iteration' +single_generated_file = build_folder + 'single_generated' configurations_folder = 'configurations/' hashconfigsort = configurations_folder + 'hashconfigsort' @@ -104,8 +116,9 @@ log_folder = 'log/' ## Programs paths parse_kconfig = 'scripts/parse_kconfig/parse' -write_config = 'scripts/write_config/write' +write_config = 'scripts/write_config/write_config' picosat = 'scripts/picosat-959/picosat' +allconfig = 'scripts/allconfig/allconfig' absroot = os.path.dirname(os.path.realpath(__file__)) diff --git a/scripts/boot.py b/scripts/boot.py index 3e715aa..dd40b35 100644 --- a/scripts/boot.py +++ b/scripts/boot.py @@ -9,22 +9,29 @@ import initialize from conf import conf from conf import sf from exceptions import MissingFile +import database -def boot(): +def boot(config, to_database = True): try: os.mkdir(sf(conf.output_folder)) except FileExistsError: pass - wd = os.getcwd() - - sprc = subprocess.Popen(conf.boot_command, - stdout = subprocess.PIPE) - with open(os.path.join(sf(conf.output_folder), utils.get_last_configuration()), "a") as f: + sprc = subprocess.Popen(conf.boot_command, stdout = subprocess.PIPE) + with open(os.path.join(sf(conf.output_folder), config.cfile), "a") as f: for linen in sprc.stdout: line = linen.decode('utf-8') if conf.boot_output: print(line, end="") f.write(line) - os.chdir(wd) + # Let user script parse double value + out = utils.callsubprocess('parse_command', conf.parse_command, + conf.parse_output, True) + value = float(out[0]) + + if to_database: + dtb = database.database() + dtb.add_measure(config.cfile, config.id, value) + + return config.cfile diff --git a/scripts/configurations.py b/scripts/configurations.py index bc73331..313108a 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -41,7 +41,7 @@ def __exec_sat__(file, args): picosat_cmd = [sf(conf.picosat), file] picosat_cmd += conf.picosat_args stdout = utils.callsubprocess('picosat', picosat_cmd, conf.picosat_output, - True, allowed_exit_codes = [10]) + True, allow_all_exit_codes = True) rtn = [] solut = [] @@ -65,13 +65,9 @@ def __exec_sat__(file, args): pass return rtn -def __write_temp_config_file__(con): +def __write_temp_config_file__(con, conf_num): # Ensure smap existence utils.build_symbol_map() - # Load variable count - with open(sf(conf.variable_count_file)) as f: - f.readline() - var_num = int(f.readline()) # Write temporally file wfile = tempfile.NamedTemporaryFile(delete=False) for s in con: @@ -80,7 +76,7 @@ def __write_temp_config_file__(con): s *= -1 else: nt = False - if s > var_num: + if s > conf_num: break; if 'NONAMEGEN' in utils.smap[s]: # ignore generated names continue @@ -147,10 +143,10 @@ def __calchash__(file): hsh = hashlib.md5(bytes(cstr, 'UTF-8')) return hsh.hexdigest() -def __register_conf__(con): +def __register_conf__(con, conf_num): dtb = database.database() # Solution to configuration - wfile = __write_temp_config_file__(con) + wfile = __write_temp_config_file__(con, conf_num) hsh = __calchash__(wfile) filen = os.path.join(sf(conf.configurations_folder), hsh) hshf = hsh @@ -164,6 +160,32 @@ def __register_conf__(con): shutil.move(wfile, filen) dtb.add_configuration(hsh, hshf) +def __generate_single__(var_num, conf_num): + if os.path.isfile(sf(conf.single_generated_file)): + return False + measure_list = [] + with open(sf(conf.measure_file), 'r') as f: + for ln in f: + measure_list.append(int(ln)) + for measure in measure_list: + tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), + sf(conf.fixed_file)), (str(measure))) + try: + confs = __exec_sat__(tfile, ['-i', '0']) + for con in confs: + __register_conf__(con, conf_num) + except exceptions.NoSolution: + pass + finally: + os.remove(tfile) + with open(sf(conf.single_generated_file), 'w') as f: + f.write("This file informs scripts, that all single selected configurations are already generated.\n") + f.write("Remove this file if you want run generating process again.") + return True + +def __generate_random__(var_num, conf_num): + # TODO + pass def generate(): """Collect boolean equations from files rules and required @@ -172,21 +194,24 @@ def generate(): # Check if rules_file exist. If it was generated. if not os.path.isfile(sf(conf.rules_file)): raise exceptions.MissingFile(conf.rules_file,"Run parse_kconfig.") - if not os.path.isfile(sf(conf.required_file)): - raise exceptions.MissingFile(conf.required_file,"Run allconfig.") + if not os.path.isfile(sf(conf.fixed_file)): + raise exceptions.MissingFile(conf.required_file,"Run allconfig and initialization process.") - # Load variable clount + # Load variable count with open(sf(conf.variable_count_file)) as f: var_num = f.readline() - tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.required_file)), ()) - try: - confs = __exec_sat__(tfile, []) - os.remove(tfile) - for con in confs: - __register_conf__(con) - except exceptions.NoSolution: - os.remove(tfile) - raise exceptions.NoSolution() + conf_num = f.readline() + + if __generate_single__(var_num, conf_num): + return + + #tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ()) + #try: + #confs = __exec_sat__(tfile, []) + #for con in confs: + #__register_conf__(con, conf_num) + #finally: + #os.remove(tfile) def compare(file1, file2): """Compared two configuration""" diff --git a/scripts/exceptions.py b/scripts/exceptions.py index 730664a..89fba0a 100644 --- a/scripts/exceptions.py +++ b/scripts/exceptions.py @@ -14,23 +14,17 @@ class NoSolution(Exception): def __str__(self): return "SAT solver found no solution. Statement is not satisfiable." -class PhaseMismatch(Exception): - def __init__(self): - pass - def __str__(self): - return "Phase in " + conf.phase_file + " is unknown." - class ConfigurationError(Exception): def __init__(self, message): self.message = message; def __str__(self): return "Configuration error: " + message -class NoApplicableSolution(Exception): +class NoApplicableConfiguration(Exception): def __init__(self): pass def __str__(self): - return "No applicable solution find. All generated solutions were already applied." + return "No applicable configuration find. All generated configurations were already applied." class ProcessFailed(Exception): def __init__(self, process, returncode): diff --git a/scripts/initialize.py b/scripts/initialize.py index f48156d..ee6c43d 100755 --- a/scripts/initialize.py +++ b/scripts/initialize.py @@ -9,12 +9,11 @@ import database from conf import conf from conf import sf import exceptions -import loop def all(): base() parse_kconfig() - gen_requred() + gen_fixed() # check if database is initialized database.database() @@ -33,17 +32,6 @@ def base(): except FileExistsError: pass - if os.path.isfile(sf(conf.phase_file)): - print("Warning: file " + conf.phase_file + " already exists. Not overwritten.") - else: - loop.phase_set(1) - - if os.path.isfile(sf(conf.iteration_file)): - print("Warning: file " + conf.iteration_file + " already exists. Not overwritten.") - else: - loop.iteration_reset() - - def parse_kconfig(): "Execute parse_kconfig in linux_sources directory." if os.path.isfile(sf(conf.symbol_map_file)) and \ @@ -52,7 +40,6 @@ def parse_kconfig(): print('Warning: parse_kconfig not executed. Files already exists.') return print('Executing parse_kconfig...') - env = dict(os.environ) wd = os.getcwd() os.chdir(sf(conf.linux_sources)) parse_kconfig_cmd = [sf(conf.parse_kconfig)] @@ -63,8 +50,18 @@ def parse_kconfig(): os.chdir(wd) -def gen_requred(): - "Generates required depenpency from dot_config file." +def __gen_allconfig_fixed__(): + wd = os.getcwd() + os.chdir(sf(conf.linux_sources)) + allconfig_cmd = [sf(conf.allconfig)] + allconfig_cmd += ['Kconfig', sf(conf.dot_config), sf(conf.dot_measure_file)] + allconfig_cmd += ['--inv'] + utils.callsubprocess("allconfig_fixed", allconfig_cmd, False, + env = utils.get_kernel_env()) + os.chdir(wd) + +def gen_fixed(): + "Generates fixed depenpency from dot_config file." print('Generating required configuration...') if not os.path.isfile(sf(conf.dot_config)): @@ -75,9 +72,10 @@ def gen_requred(): srmap = {value:key for key, value in utils.smap.items()} # swap dictionary shutil.copy(sf(conf.dot_config), sf(conf.dot_config_back_file)) + __gen_allconfig_fixed__() with open(sf(conf.dot_config), 'r') as f: - with open(sf(conf.required_file), 'w') as freq: + with open(sf(conf.fixed_file), 'w') as ffix: for line in f: if (line[0] == '#') or (not '=' in line): continue @@ -85,9 +83,18 @@ def gen_requred(): if (line[indx + 1] == 'y'): if line[7:indx] == "MODULES": # exception if modules set raise exceptions.ConfigurationError("Fixed kernel configuration must have MODULES disabled.") - freq.write(str(srmap[line[7:indx]]) + "\n") + ffix.write(str(srmap[line[7:indx]]) + "\n") elif (line[indx + 1] == 'n' or line[indx + 1] == 'm'): - freq.write("-" + str(srmap[line[7:indx]]) + "\n") + ffix.write("-" + str(srmap[line[7:indx]]) + "\n") + with open(sf(conf.dot_measure_file), 'r') as f: + with open(sf(conf.measure_file), 'w') as fmes: + for line in f: + if (line[0] == '#') or (not '=' in line): + continue + indx = line.index('=') + if line[7:indx] == "MODULES": + raise exceptions.ConfigurationError("Can't measure configuraion option MODULES. Not supported.") + fmes.write(str(srmap[line[7:indx]]) + "\n") ################################################################################# diff --git a/scripts/loop.py b/scripts/loop.py index fc15d8a..2f008dc 100755 --- a/scripts/loop.py +++ b/scripts/loop.py @@ -4,6 +4,7 @@ import sys import subprocess import signal from threading import Thread +from threading import Lock from conf import conf from conf import sf @@ -12,145 +13,100 @@ import configurations import kernel import boot import exceptions +import database -def step(): - phs = phase_get() - if phs == 0 or phs == 1: - phase_message(1) - initialize.all() - phase_set(2) - elif phs == 2: - phase_message(2) - phase_set(3) - elif phs == 3: - phase_message(3) - try: - configurations.apply() - except exceptions.NoApplicableSolution: - try: - os.mkdir(sf(conf.result_folder)) - except FileExistsError: - pass - print('\nAll done.') - exit(0) - phase_set(4) - elif phs == 4: - phase_message(4) - phase_set(5) - elif phs == 5: - phase_message(5) - try: - kernel.config() - except exceptions.ConfigurationError: - if not conf.ignore_misconfig: - print("Configuration mismatch. Exiting.") - sys.exit(-2) - phase_set(6) - elif phs == 6: - phase_message(6) - if conf.only_config: - phase_set(3) - else: - phase_set(7) - elif phs == 7: - phase_message(7) - kernel.make() - phase_set(8) - elif phs == 8: - phase_message(8) - phase_set(9) - elif phs == 9: - phase_message(9) - boot.boot() - phase_set(10) - elif phs == 10: - phase_message(10) - phase_set(3) +__confs_unmeasured__ = [] -# Phase # -phases = ("Not Initialized", #0 - "Initializing", #1 - "Initialized", #2 - "Solution applying", #3 - "Solution applied", #4 - "Kernel configuration", #5 - "Kernel configured", #6 - "Kernel build", #7 - "Kernel built", #8 - "System boot", #9 - "Benchmark successful" #10 - ) +def prepare(): + """Prepare for measuring + Outcome is Linux image for generated configuration.""" + global __confs_unmeasured__ + if len(__confs_unmeasured__) == 0: + dtb = database.database() + confs = dtb.get_unmeasured() + if len(confs) == 0: + configurations.generate() + confs = dtb.get_unmeasured() + if len(confs) == 0: + raise exceptions.NoApplicableConfiguration() + __confs_unmeasured__ = list(confs) + con = __confs_unmeasured__.pop() + kernel.config(con.cfile) + img = kernel.make(con.hash) + print("Prepared image: " + img) + return img, con -def phase_get(): +def measure(kernelimg, con): try: - with open(sf(conf.phase_file)) as f: - txtPhase = f.readline().rstrip() - if not txtPhase in phases: - raise PhaseMismatch() - return phases.index(txtPhase) + os.remove(sf(conf.jobfolder_linux_image)) except FileNotFoundError: - return 0 - -def phase_set(phs): - # TODO - try: - global thr - if thr.term: - return - except NameError: pass - with open(sf(conf.phase_file), 'w') as f: - f.write(phases[phs]) + os.symlink(os.path.join(sf(conf.build_folder), kernelimg), + sf(conf.jobfolder_linux_image)) + boot.boot(con) + print("Configuration '" + con.hash + "' measured.") -def phase_message(phs): - "Prints message signaling running phase_" - print("-- " + phases[phs]) +# Threads # +__terminate__ = False +class mainThread(Thread): + def run(self): + if conf.single_loop: + img, config = prepare() + measure(img, config) + else: + while not __terminate__: + img, config = prepare() + measure(img, config) -# Iteration # -def iteration_reset(): - with open(sf(conf.iteration_file), 'w') as f: - f.write('0') +# Multithread section # +__conflist__ = [] +__listlock__ = Lock() -def iteration_inc(): - with open(sf(conf.iteration_file), 'r') as f: - it = int(f.readline()) - it += 1 - with open(sf(conf.iteration_file), 'w') as f: - f.write(str(it)) +class prepareThread(Thread): + def __init__(self, name='prepare'): + Thread.__init__(self, name=name) + def run(self): + __listlock__.aquire() + while not __terminate__ and len(__conflist__) <= conf.multithread_buffer: + __listlock__.release() + config = prepare() + __listlock__.aquire() + __conflist__.append(config) + if not __measurethread__.isActive(): + __measurethread__.start() + __listlock__.release() -# Thread # -class mainThread(Thread): - def __init__(self, name): +class measureThread(Thread): + def __init__(self, name='measure'): Thread.__init__(self, name=name) - self.term = False def run(self): - if conf.step_by_step: - step() - elif conf.single_loop: - while not phase_get() == 2: - step() - step() - while not phase_get() == 2: - step() - else: - while not self.term: - step() + __listlock__.aquire() + while not __terminate__ and len(__conflist__) > 0: + config = __conflist__[0] + del __conflist__[0] + __listlock__.release() + if not __preparethread__.isActive(): + __preparethread__.start() + measure(config) + __listlock__.aquire() + __listlock__.release() -def loop_term(): - global thr - thr.term = True +__preparethread__ = prepareThread() +__measurethread__ = measureThread() +# Start and sigterm handler # def sigterm_handler(_signo, _stack_frame): - loop_term() + __terminate__ = True def loop(): + initialize.all() global thr - thr = mainThread("thred") + thr = mainThread() thr.start() try: thr.join() except KeyboardInterrupt: - loop_term() + __terminate__ = True ################################################################################# diff --git a/scripts/test.py b/scripts/test.py index 5b94df2..b01a8b6 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -7,15 +7,25 @@ from conf import sf import initialize import kernel import boot +import database def test(): initialize.base() initialize.parse_kconfig() - initialize.gen_requred() # Call this to check initial solution + print("-- Make --") conf.kernel_make_output = True - kernel.make() + img = kernel.make('test') + try: + os.remove(sf(conf.jobfolder_linux_image)) + except FileNotFoundError: + pass + os.symlink(os.path.join(sf(conf.build_folder), img), + sf(conf.jobfolder_linux_image)) conf.boot_output = True - boot.boot() + conf.parse_output = True + print("-- Boot --") + config = database.Config('0', 'test', img) + boot.boot(config, False) ################################################################################# diff --git a/scripts/utils.py b/scripts/utils.py index d138279..7a7a79d 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -29,7 +29,8 @@ def build_symbol_map(): def callsubprocess(process_name, process, show_output = True, - return_output = False, env=os.environ, allowed_exit_codes = [0]): + return_output = False, env=os.environ, allowed_exit_codes = [0], + allow_all_exit_codes = False): sprc = subprocess.Popen(process, stdout = subprocess.PIPE, env = env) try: @@ -51,7 +52,7 @@ def callsubprocess(process_name, process, show_output = True, rtn.append(line.rstrip()) rtncode = sprc.wait() - if rtncode not in allowed_exit_codes: + if rtncode not in allowed_exit_codes and not allow_all_exit_codes: raise exceptions.ProcessFailed(process, rtncode) return rtn @@ -59,73 +60,3 @@ def get_kernel_env(): env = dict(os.environ) env.update(conf.kernel_env) return env - - -def hash_config(cf): - """Hashes configuration using MD5 hash. - """ - try: - cf.remove(0) - except ValueError: - pass - str = "" - for c in cf: - if c < 0: - str += '-' - else: - str += '+' - hsh = hashlib.md5(bytes(str, sys.getdefaultencoding())) - return hsh.hexdigest() - -def config_strtoint(str, full): - """Reads list of configured symbols from string - """ - rtn = [] - if full: - for s in str.rstrip().split(sep=' '): - rtn.append(int(s)) - else: - count = 0 - with open(sf(conf.variable_count_file)) as f: - f.readline() - count = int(f.readline()) - for s in str.rstrip().split(sep=' '): - val = int(s) - if abs(val) <= count: - rtn.append(val) - else: - break; - try: - rtn.remove(0) - except ValueError: - pass - return rtn - -def get_config_from_hash(hash): - with open(sf(conf.config_map_file), "r") as f: - for line in f: - w = line.rstrip().split(sep=':') - if w[0] == hash: - return config_strtoint(w[1], True) - return None - -def get_last_configuration(): - hsh = "" - try: - with open(sf(conf.config_solved_file), "r") as f: - for line in f: - sline = line.rstrip() - if sline != '': - hsh = sline - except FileNotFoundError: - try: - with open(sf(conf.config_map_file), "r") as f: - w = f.readline().split(sep=':') - hsh = w[0] - except FileNotFoundError: - pass - - if hsh != '': - return hsh - else: - return 'NoConfig' |