From 86eb2c8945b952492a99f774262a2aacaaf58776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 10:46:27 +0200 Subject: Single configuration generation ensured to generate only single configuration per execution --- scripts/configurations.py | 62 ++++++++++++++++++++++++++--------------------- scripts/exceptions.py | 5 ++++ 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index 5a04f38..3fb22d9 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -161,31 +161,41 @@ def __register_conf__(con, conf_num, generator): dtb.add_configuration(hsh, hshf, generator) def __generate_single__(var_num, conf_num): - if os.path.isfile(sf(conf.single_generated_file)): + measure_list = set() + if not os.path.isfile(sf(conf.single_generated_file)): + with open(sf(conf.measure_file), 'r') as fi: + for ln in fi: + measure_list.add(int(ln)) + else: + with open(sf(conf.single_generated_file), 'r') as f: + for ln in f: + measure_list.append(int(ln)) + if measure_list: return False - measure_list = [] - with open(sf(conf.measure_file), 'r') as f: - for ln in f: - measure_list.append(int(ln)) - for measure in measure_list: - tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), - sf(conf.fixed_file)), (str(measure))) - try: - confs = __exec_sat__(tfile, ['-i', '0']) - for con in confs: - __register_conf__(con, conf_num, 'single-sat') - except exceptions.NoSolution: - pass - finally: - os.remove(tfile) - with open(sf(conf.single_generated_file), 'w') as f: - f.write("This file informs scripts, that all single selected configurations are already generated.\n") - f.write("Remove this file if you want run generating process again.") - return True + tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), + sf(conf.fixed_file)), (str(measure_list.pop()))) + try: + confs = __exec_sat__(tfile, ['-i', '0']) + for con in confs: + __register_conf__(con, conf_num, 'single-sat') + except exceptions.NoSolution: + pass + finally: + os.remove(tfile) + with open(sf(conf.single_generated_file), 'w') as fo: + fo.writelines(measure_list) + return True def __generate_random__(var_num, conf_num): # TODO - pass + #tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ()) + #try: + #confs = __exec_sat__(tfile, []) + #for con in confs: + #__register_conf__(con, conf_num) + #finally: + #os.remove(tfile) + return False def generate(): """Collect boolean equations from files rules and required @@ -204,14 +214,10 @@ def generate(): if __generate_single__(var_num, conf_num): return + elif __generate_random__(var_num, conf_num): + return - #tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ()) - #try: - #confs = __exec_sat__(tfile, []) - #for con in confs: - #__register_conf__(con, conf_num) - #finally: - #os.remove(tfile) + raise exceptions.NoNewConfiguration() def compare(file1, file2): """Compared two configuration""" diff --git a/scripts/exceptions.py b/scripts/exceptions.py index 89fba0a..ac7f093 100644 --- a/scripts/exceptions.py +++ b/scripts/exceptions.py @@ -20,6 +20,11 @@ class ConfigurationError(Exception): def __str__(self): return "Configuration error: " + message +class NoNewConfiguration(Exception): + def __init__(self): + pass + def __str__(self): + return "No new configuration generated" class NoApplicableConfiguration(Exception): def __init__(self): pass -- cgit v1.2.3 From 8a3d7b8ed5b559c3777b9ee4715361d2dc01da07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 10:47:27 +0200 Subject: Change default database configuration Database configuration was set as it was used during development and testing. Changed to some of default not usable values. --- conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf.py b/conf.py index d75b5dd..583981b 100644 --- a/conf.py +++ b/conf.py @@ -37,13 +37,13 @@ picosat_args = [] # db_database # Database in PostgreSQL to be used for this tools -db_database = 'lcp' +db_database = 'linux-conf-perf' # db_user # Define PostgreSQL user -db_user = 'cynerd' +db_user = 'user' # db_password # Define PostrgreSQL user password -db_password = 'cynerd' +db_password = 'password' # db_host # Address of PostgreSQL database server db_host = 'localhost' -- cgit v1.2.3 From b88557fa9e30b57e616635e3037ea5b56f7d4d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Thu, 13 Aug 2015 09:59:33 +0200 Subject: Change hash calculating function Previous method of calculating hash wasn't work (wouldn't give same result for same configuration) if there would be added new configuration options to hashconfigsort file. --- scripts/configurations.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index 3fb22d9..ba326ea 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -119,11 +119,9 @@ def __calchash__(file): for c in csort: try: if con[c]: - cstr += '+' - else: - cstr += '-' + cstr += c except ValueError: - cstr += '0' + pass # Add missing csortfile = open(sf(conf.hashconfigsort), 'a'); @@ -135,9 +133,7 @@ def __calchash__(file): csort.append(key) csortfile.write(key + '\n') if val: - cstr += '+' - else: - cstr += '-' + cstr += key csortfile.close() hsh = hashlib.md5(bytes(cstr, 'UTF-8')) -- cgit v1.2.3 From 78e08ef8cd15a60f19b15a9b5f1dfcdf57767721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Tue, 11 Aug 2015 19:43:30 +0200 Subject: Add measure tool desription to database --- conf.py | 4 ++++ scripts/database.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/conf.py b/conf.py index 583981b..965d2f4 100644 --- a/conf.py +++ b/conf.py @@ -31,6 +31,10 @@ boot_command = ['echo', 'bootit'] # Command to parse double value from boot output parse_command = ['echo', '0'] +# measurement_identifier +# Identifier of measurement can consist of measure tool name and version +measure_identifier = 'cyclictest-v0.92' + # picosat_args # Additional arguments passed to PicoSAT. picosat_args = [] diff --git a/scripts/database.py b/scripts/database.py index 16556a8..eef118d 100644 --- a/scripts/database.py +++ b/scripts/database.py @@ -102,14 +102,14 @@ class database: def add_measure(self, mfile, conf_id, value = None): "Add measurement." ps = self.db.prepare("""INSERT INTO measure - (conf, mfile, value, mtime, toolgit, linuxgit) + (conf, mfile, value, mtime, toolgit, linuxgit, measurement) VALUES - ($1, $2, $3, $4, $5, $6); + ($1, $2, $3, $4, $5, $6, $7); """) gt = self.check_toolsgit() lgt = self.check_linuxgit() tm = datetime.datetime.now() - ps(conf_id, mfile, value, tm, gt, lgt) + ps(conf_id, mfile, value, tm, gt, lgt, conf.measure_identifier) def update_measure(self, measure_id, value): "Update measured value" -- cgit v1.2.3 From 3adf6c6ffefbb1b0850401f3415d321ef2d2e42c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 15:43:06 +0200 Subject: Fix bug in configuration.py Type inserted to method is str, but threated as int. --- scripts/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index ba326ea..c58f3f5 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -76,7 +76,7 @@ def __write_temp_config_file__(con, conf_num): s *= -1 else: nt = False - if s > conf_num: + if s > int(conf_num): break; if 'NONAMEGEN' in utils.smap[s]: # ignore generated names continue -- cgit v1.2.3 From df5b3585299628def438de1b7569b0677491ea34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 15:48:19 +0200 Subject: Fix if generated existing configuration When some generator tries to register already generated solution, it shouldn't be registered. --- scripts/configurations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/configurations.py b/scripts/configurations.py index c58f3f5..2909578 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -149,6 +149,7 @@ def __register_conf__(con, conf_num, generator): if os.path.isfile(filen): if compare(filen, wfile): print("I: Generated existing configuration.") + return else: print("W: Generated configuration with collision hash.") # TODO this might have to be tweaked -- cgit v1.2.3 From 3ee62e5d11b8516bda319e6e427e35dd4509f6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 16:41:59 +0200 Subject: Fix problem with wrongly printed first line in CNF file --- scripts/configurations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index 2909578..e3a7d0b 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -206,8 +206,8 @@ def generate(): # Load variable count with open(sf(conf.variable_count_file)) as f: - var_num = f.readline() - conf_num = f.readline() + var_num = f.readline().rstrip() + conf_num = f.readline().rstrip() if __generate_single__(var_num, conf_num): return -- cgit v1.2.3 From 523c57cee7c9d4909788c85cc392c60e8c056e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 17:28:00 +0200 Subject: Fix bug in configurations loading Loading configuration load all configuration options as True. --- scripts/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index e3a7d0b..097f501 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -100,7 +100,7 @@ def __load_config_file__(file): if (ln[indx + 1] == 'y'): rtn[ln[7:indx]] = True else: - rtn[ln[7:indx]] = True + rtn[ln[7:indx]] = False return rtn def __calchash__(file): -- cgit v1.2.3 From dfffbc03559941bcb3a430cb7319194d31685323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 18:12:59 +0200 Subject: Fix wrong configuration reading from database --- scripts/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/database.py b/scripts/database.py index eef118d..6a4d0a1 100644 --- a/scripts/database.py +++ b/scripts/database.py @@ -132,8 +132,8 @@ class database: def get_unmeasured(self): "Returns list of all unmeasured configurations." - ps = self.db.prepare("""SELECT * FROM configurations - WHERE NOT EXISTS + ps = self.db.prepare("""SELECT id, hash, cfile FROM configurations + WHERE id NOT IN (SELECT conf FROM measure) """) rtn = [] -- cgit v1.2.3 From 89aceeeafc21816b527c82c133744438eaa48f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Wed, 12 Aug 2015 20:31:28 +0200 Subject: Catch NoApplicableConfiguration exception in prepare thread --- scripts/loop.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/loop.py b/scripts/loop.py index fc0ea44..aecda39 100755 --- a/scripts/loop.py +++ b/scripts/loop.py @@ -68,7 +68,10 @@ class prepareThread(Thread): __listlock__.aquire() while not __terminate__ and len(__conflist__) <= conf.multithread_buffer: __listlock__.release() - config = prepare() + try: + config = prepare() + except exceptions.NoApplicableConfiguration: + return __listlock__.aquire() __conflist__.append(config) if not __measurethread__.isActive(): -- cgit v1.2.3 From 96825e484149a44d17e2f2330c8fe2ee4563259c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Thu, 13 Aug 2015 11:04:31 +0200 Subject: Implement random generation of configurations Configuration are generated with random assumptions. This should generate different configuration for every SAT solver execution. --- scripts/configurations.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index 097f501..0b73a53 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -149,13 +149,14 @@ def __register_conf__(con, conf_num, generator): if os.path.isfile(filen): if compare(filen, wfile): print("I: Generated existing configuration.") - return + return False else: print("W: Generated configuration with collision hash.") # TODO this might have to be tweaked raise Exception() shutil.move(wfile, filen) dtb.add_configuration(hsh, hshf, generator) + return True def __generate_single__(var_num, conf_num): measure_list = set() @@ -184,15 +185,15 @@ def __generate_single__(var_num, conf_num): return True def __generate_random__(var_num, conf_num): - # TODO - #tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ()) - #try: - #confs = __exec_sat__(tfile, []) - #for con in confs: - #__register_conf__(con, conf_num) - #finally: - #os.remove(tfile) - return False + tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ()) + try: + confs = __exec_sat__(tfile, ['-i', '3']) + for con in confs: + if not __register_conf__(con, conf_num, 'random-sat'): + __generate_random__(var_num, conf_num) + finally: + os.remove(tfile) + return True def generate(): """Collect boolean equations from files rules and required -- cgit v1.2.3 From 88bce0d3359f570388c21e04c0556e6736afc28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Thu, 13 Aug 2015 11:19:09 +0200 Subject: Fix thread cooperation in loop Measure function requires img and config as atributes. But only config was provided. --- scripts/loop.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/loop.py b/scripts/loop.py index aecda39..9cc8ddb 100755 --- a/scripts/loop.py +++ b/scripts/loop.py @@ -58,7 +58,7 @@ class mainThread(Thread): measure(img, config) # Multithread section # -__conflist__ = [] +__conflist__ = set() __listlock__ = Lock() class prepareThread(Thread): @@ -69,11 +69,11 @@ class prepareThread(Thread): while not __terminate__ and len(__conflist__) <= conf.multithread_buffer: __listlock__.release() try: - config = prepare() + img, config = prepare() except exceptions.NoApplicableConfiguration: return __listlock__.aquire() - __conflist__.append(config) + __conflist__.add((img, config)) if not __measurethread__.isActive(): __measurethread__.start() __listlock__.release() @@ -84,12 +84,11 @@ class measureThread(Thread): def run(self): __listlock__.aquire() while not __terminate__ and len(__conflist__) > 0: - config = __conflist__[0] - del __conflist__[0] + img, config = __conflist__.pop() __listlock__.release() if not __preparethread__.isActive(): __preparethread__.start() - measure(config) + measure(img, config) __listlock__.aquire() __listlock__.release() -- cgit v1.2.3 From 5bf8e17c83d4b5500fd81edaee5c495161e1f0ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Thu, 13 Aug 2015 11:42:51 +0200 Subject: Fix generate single configuration This fixes and ensures that every call generates configuration. Unless False is returned. --- scripts/configurations.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/configurations.py b/scripts/configurations.py index 0b73a53..a9ede05 100644 --- a/scripts/configurations.py +++ b/scripts/configurations.py @@ -167,21 +167,22 @@ def __generate_single__(var_num, conf_num): else: with open(sf(conf.single_generated_file), 'r') as f: for ln in f: - measure_list.append(int(ln)) - if measure_list: + measure_list.add(int(ln)) + if not measure_list: return False tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), (str(measure_list.pop()))) + with open(sf(conf.single_generated_file), 'w') as fo: + fo.writelines(measure_list) try: confs = __exec_sat__(tfile, ['-i', '0']) for con in confs: __register_conf__(con, conf_num, 'single-sat') except exceptions.NoSolution: - pass + os.remove(tfile) + return __generate_single__(var_num, conf_num) finally: os.remove(tfile) - with open(sf(conf.single_generated_file), 'w') as fo: - fo.writelines(measure_list) return True def __generate_random__(var_num, conf_num): -- cgit v1.2.3