From 3fb56b4760b0485cf0872100f04b0a5a51f52e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Mon, 17 Aug 2015 18:35:12 +0200 Subject: Generated configuration is now fully stored to database Managing configurations in files and in database could cause inconsistence. Adding all generated configurations to database allow us to clean project files without loosing data. --- scripts/configurations.py | 101 ++++++++++++++++++++++++++-------------------- scripts/database.py | 24 ++++++++--- scripts/databaseinit.sql | 9 ++++- scripts/kernel.py | 7 +++- scripts/loop.py | 3 +- 5 files changed, 91 insertions(+), 53 deletions(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index 3f2d15e..b346169 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -65,11 +65,11 @@ def __exec_sat__(file, args): pass return rtn -def __write_temp_config_file__(con, conf_num): +def __txt_config__(con, conf_num): # Ensure smap existence utils.build_symbol_map() # Write temporally file - wfile = tempfile.NamedTemporaryFile(delete=False) + txt = '' for s in con: if s < 0: nt = True @@ -80,41 +80,44 @@ def __write_temp_config_file__(con, conf_num): break; if 'NONAMEGEN' in utils.smap[s]: # ignore generated names continue - wfile.write(bytes('CONFIG_' + utils.smap[s] + '=', - sys.getdefaultencoding())) + txt += 'CONFIG_' + utils.smap[s] + '=' if not nt: - wfile.write(bytes('y', sys.getdefaultencoding())) + txt += 'y' else: - wfile.write(bytes('n', sys.getdefaultencoding())) - wfile.write(bytes('\n', sys.getdefaultencoding())) + txt += 'n' + txt += '\n' + return txt + +def __write_temp_config_file__(con, conf_num): + wfile = tempfile.NamedTemporaryFile(delete=False) + txt = __txt_config__(con, conf_num) + wfile.write(bytes(txt, sys.getdefaultencoding())) wfile.close() return wfile.name -def __load_config_file__(file): +def __load_config_text__(txt): rtn = dict() - with open(file, 'r') as f: - for ln in f: - if ln[0] == '#' or not '=' in ln: - continue - indx = ln.index('=') - if (ln[indx + 1] == 'y'): - rtn[ln[7:indx]] = True - else: - rtn[ln[7:indx]] = False + for ln in txt: + if ln[0] == '#' or not '=' in ln: + continue + indx = ln.index('=') + if (ln[indx + 1] == 'y'): + rtn[ln[7:indx]] = True + else: + rtn[ln[7:indx]] = False return rtn -def __calchash__(file): - """Calculates hash from configuration file""" - # Build hashconfigsort - csort = [] - try: - with open(conf.hashconfigsort, 'r') as f: - for ln in f: - csort.append(ln.rstrip()) - except FileNotFoundError: - pass - con = __load_config_file__(file) +def __load_config_file__(file): + f = open(file, 'r') + rtn = __load_config_text__(f) + f.close() + return rtn + +def __calchash__(con): + dt = database.database() + csort = dt.get_configsort() + cstr = "" for c in csort: try: @@ -124,38 +127,41 @@ def __calchash__(file): pass # Add missing - csortfile = open(sf(conf.hashconfigsort), 'a'); for key, val in con.items(): try: csort.index(key) except ValueError: indx = len(csort) csort.append(key) - csortfile.write(key + '\n') + dt.add_configsort(key) if val: cstr += key - csortfile.close() hsh = hashlib.md5(bytes(cstr, 'UTF-8')) return hsh.hexdigest() + +def __calchash_file__(file): + """Calculates hash from configuration file""" + con = __load_config_file__(file) + return __calchash__(con) + def __register_conf__(con, conf_num, generator): dtb = database.database() # Solution to configuration - wfile = __write_temp_config_file__(con, conf_num) - hsh = __calchash__(wfile) - filen = os.path.join(sf(conf.configurations_folder), hsh) - hshf = hsh - if os.path.isfile(filen): - if compare(filen, wfile): + txtconfig = __txt_config__(con, conf_num) + hsh = __calchash__(con) + cconf = dtb.get_configration(hsh) + for cc in cconf: + print('hash: ' + hsh) + if compare_text(cc, txtconfig): print("I: Generated existing configuration.") return False else: print("W: Generated configuration with collision hash.") # TODO this might have to be tweaked raise Exception() - shutil.move(wfile, filen) - dtb.add_configuration(hsh, hshf, generator) + dtb.add_configuration(hsh, txtconfig, generator) return True def __generate_single__(var_num, conf_num): @@ -218,11 +224,7 @@ def generate(): raise exceptions.NoNewConfiguration() -def compare(file1, file2): - """Compared two configuration""" - conf1 = __load_config_file__(file1) - conf2 = __load_config_file__(file2) - +def compare(conf1, conf2): # This is not exactly best comparison method for key, val in conf1.items(): try: @@ -237,3 +239,14 @@ def compare(file1, file2): except ValueError: return False return True + +def compare_text(text1, text2): + conf1 = __load_config_text__(text1) + conf2 = __load_config_text__(text2) + return compare_file(conf1, conf2) + +def compare_file(file1, file2): + """Compared two configuration""" + conf1 = __load_config_file__(file1) + conf2 = __load_config_file__(file2) + return compare_file(conf1, conf2) diff --git a/scripts/database.py b/scripts/database.py index 6a4d0a1..c4f0a60 100644 --- a/scripts/database.py +++ b/scripts/database.py @@ -19,7 +19,7 @@ def __git_commit__(): def __timestamp__(): return datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S') -Config = collections.namedtuple('Config', 'id hash cfile') # Named tuple for configuration +Config = collections.namedtuple('Config', 'id hash config') # Named tuple for configuration Measure = collections.namedtuple('Measure', 'id conf_id mfile value') # Named tuple for measurement class database: @@ -78,21 +78,21 @@ class database: ps(ds, cm) return self.check_linuxgit() - def add_configuration(self, hash, cfile, generator): + def add_configuration(self, hash, txtconfig, generator): "Add configuration to database." ps = self.db.prepare("""INSERT INTO configurations - (hash, cfile, gtime, toolgit, linuxgit, generator) + (hash, config, gtime, toolgit, linuxgit, generator) VALUES ($1, $2, $3, $4, $5, $6); """) gt = self.check_toolsgit() lgt = self.check_linuxgit() tm = datetime.datetime.now() - ps(hash, cfile, tm, gt, lgt, generator) + ps(hash, txtconfig, tm, gt, lgt, generator) def get_configration(self, hash): "Return configration id for inserted hash." - ps = self.db.prepare("""SELECT id, cfile FROM configurations + ps = self.db.prepare("""SELECT id, config FROM configurations WHERE hash = $1""") rtn = [] for dt in ps(hash): @@ -140,3 +140,17 @@ class database: for dt in ps(): rtn.append(Config(dt[0], dt[1], dt[2])) return rtn + + def add_configsort(self, configopt): + "Add configuration option to sorted list" + ps = self.db.prepare("""INSERT INTO configopt + (configopt) VALUES ($1) + """) + ps() + + def get_configsort(self): + "Returns sorted list of all configuration options" + ps = self.db.prepare("""SELECT configopt FROM configopt + ORDER BY id ASC + """) + return ps() diff --git a/scripts/databaseinit.sql b/scripts/databaseinit.sql index 1e34c75..d1ab3e2 100644 --- a/scripts/databaseinit.sql +++ b/scripts/databaseinit.sql @@ -17,7 +17,7 @@ CREATE TABLE configurations ( id BIGSERIAL PRIMARY KEY, -- Id hash char(32) NOT NULL, -- Hash of configuration generator TEXT NOT NULL, -- Text identificator of configure generation method - cfile TEXT NOT NULL, -- File path with configuration + config TEXT NOT NULL, -- Full configuration in text form gtime timestamp NOT NULL, -- Time and date of generation linuxgit BIGINT REFERENCES linuxgit (id), -- Reference to git version of Linux toolgit BIGINT REFERENCES toolsgit (id) -- Reference to git version of tools @@ -34,3 +34,10 @@ CREATE TABLE measure ( linuxgit BIGINT REFERENCES linuxgit (id), -- Reference to git version of Linux toolgit BIGINT REFERENCES toolsgit (id) -- Reference to git version of tools ); + +-- In this table are sorted all used configuration options +-- Order in this table is fundamental for configuration hash calculation +CREATE TABLE configopt ( + id BIGSERIAL PRIMARY KEY, -- Id + configopt TEXT NOT NULL -- Name of configuration option +); diff --git a/scripts/kernel.py b/scripts/kernel.py index 092de1e..c41fe1f 100644 --- a/scripts/kernel.py +++ b/scripts/kernel.py @@ -2,15 +2,18 @@ import os import sys import subprocess import shutil +import tempfile from conf import conf from conf import sf import exceptions import utils -def config(cfile): +def config(txtconfig): + "Apply text configuration to kernel folder" + infile = tempfile.NamedTemporaryFile() + infile.write(bytes(txtconfig, sys.getdefaultencoding())) wd = os.getcwd() - infile = os.path.join(sf(conf.configurations_folder), cfile) os.chdir(sf(conf.linux_sources)) try: utils.callsubprocess('write_config', [sf(conf.write_config), infile], diff --git a/scripts/loop.py b/scripts/loop.py index 9cc8ddb..2a3739e 100755 --- a/scripts/loop.py +++ b/scripts/loop.py @@ -20,6 +20,7 @@ __confs_unmeasured__ = [] def prepare(): """Prepare for measuring Outcome is Linux image for generated configuration.""" + print("Preparing new image.") global __confs_unmeasured__ if len(__confs_unmeasured__) == 0: dtb = database.database() @@ -31,7 +32,7 @@ def prepare(): raise exceptions.NoApplicableConfiguration() __confs_unmeasured__ = list(confs) con = __confs_unmeasured__.pop() - kernel.config(con.cfile) + kernel.config(con.config) img = kernel.make(con.hash) print("Prepared image: " + img) return img, con -- cgit v1.2.3