Scripts changed to use database.

Also initial implementation of multithread execution.

A lot of functionality changed.
Phases removed.
Output parsing is now part of measure (boot) process.
Utils cleared.
Add dot_measure file for inverted dot_config.
Configuration generating is now prepared for multiple generating types. Fist implemented is generating configurations with single selected configuration.

Test is modified to be compatible with new changes.

8 files changed, 196 insertions, 253 deletions

diff --git a/conf.py b/conf.py
index ad8f49f..d75b5dd 100644
--- a/conf.py
+++ b/conf.py
@@ -25,7 +25,11 @@ novaboot_args = ['--qemu=qemu-system-x86_64']
 nbscript = 'scripts/nbscript'
 # boot_command
 # Command executed for booting. Output of this command is saved to output folder.
-boot_command = ['scripts/novaboot/novaboot', nbscript] + novaboot_args
+boot_command = ['echo', 'bootit']
+
+# parse_command
+# Command to parse double value from boot output
+parse_command = ['echo', '0']
 
 # picosat_args
 # Additional arguments passed to PicoSAT.
@@ -47,6 +51,13 @@ db_host = 'localhost'
 # Port of PotgreSQL database server
 db_port = 5432
 
+# multithread
+# Define if measurement and kernel build should be executed in parallel.
+multithread = False
+# multithread_buffer
+# Defines maximal number of buffered configurations before generating is suspended.
+multithread_buffer = 32
+
 # git_describe_cmd
 # Command used for getting tools version and status from git
 git_describe_cmd = ['git', 'describe', '--always', '--tags', '--dirty']
@@ -63,9 +74,9 @@ picosat_output = False
 kernel_config_output = True
 kernel_make_output = True
 boot_output = True
+parse_output = False
 
 ## Configs for debugging
-step_by_step = False # Executes only single step and exits.
 single_loop = False # Executes only one loop and exits.
 only_config = False # Executes only to configuration phase. Building and booting phases are skipped.
 ignore_misconfig = False # Ignore if configuration wasn't applied correctly.
@@ -87,13 +98,14 @@ buildroot_initram = 'buildroot/output/images/rootfs.cpio.gz'
 
 build_folder = 'jobfiles/'
 jobfolder_linux_image = build_folder + 'linuxImage'
-phase_file = build_folder + 'phase'
 symbol_map_file = build_folder + 'symbol_map' # Also defined in parse_kconfig
 rules_file = build_folder + 'rules' # Also defined in parse_kconfig
 variable_count_file = build_folder + 'variable_count' # Also defined in parse_kconfig
-required_file = build_folder + 'required'
+fixed_file = build_folder + 'fixed'
+measure_file = build_folder + 'measure'
+dot_measure_file = build_folder + 'dot_measure'
 dot_config_back_file = build_folder + 'dot_config_back'
-iteration_file = build_folder + 'iteration'
+single_generated_file = build_folder + 'single_generated'
 
 configurations_folder = 'configurations/'
 hashconfigsort = configurations_folder + 'hashconfigsort'
@@ -104,8 +116,9 @@ log_folder = 'log/'
 
 ## Programs paths
 parse_kconfig = 'scripts/parse_kconfig/parse'
-write_config = 'scripts/write_config/write'
+write_config = 'scripts/write_config/write_config'
 picosat = 'scripts/picosat-959/picosat'
+allconfig = 'scripts/allconfig/allconfig'
 
 
 absroot = os.path.dirname(os.path.realpath(__file__))
diff --git a/scripts/boot.py b/scripts/boot.py
index 3e715aa..dd40b35 100644
--- a/scripts/boot.py
+++ b/scripts/boot.py
@@ -9,22 +9,29 @@ import initialize
 from conf import conf
 from conf import sf
 from exceptions import MissingFile
+import database
 
-def boot():
+def boot(config, to_database = True):
 	try:
 		os.mkdir(sf(conf.output_folder))
 	except FileExistsError:
 			pass
 
-	wd = os.getcwd()
-
-	sprc = subprocess.Popen(conf.boot_command,
-			stdout = subprocess.PIPE)
-	with open(os.path.join(sf(conf.output_folder), utils.get_last_configuration()), "a") as f:
+	sprc = subprocess.Popen(conf.boot_command, stdout = subprocess.PIPE)
+	with open(os.path.join(sf(conf.output_folder), config.cfile), "a") as f:
 		for linen in sprc.stdout:
 			line = linen.decode('utf-8')
 			if conf.boot_output:
 				print(line, end="")
 			f.write(line)
 
-	os.chdir(wd)
+	# Let user script parse double value
+	out = utils.callsubprocess('parse_command', conf.parse_command,
+			conf.parse_output, True)
+	value = float(out[0])
+
+	if to_database:
+		dtb = database.database()
+		dtb.add_measure(config.cfile, config.id, value)
+
+	return config.cfile
diff --git a/scripts/configurations.py b/scripts/configurations.py
index bc73331..313108a 100644
--- a/scripts/configurations.py
+++ b/scripts/configurations.py
@@ -41,7 +41,7 @@ def __exec_sat__(file, args):
 	picosat_cmd = [sf(conf.picosat), file]
 	picosat_cmd += conf.picosat_args
 	stdout = utils.callsubprocess('picosat', picosat_cmd, conf.picosat_output,
-			True, allowed_exit_codes = [10])
+			True, allow_all_exit_codes = True)
 
 	rtn = []
 	solut = []
@@ -65,13 +65,9 @@ def __exec_sat__(file, args):
 		pass
 	return rtn
 
-def __write_temp_config_file__(con):
+def __write_temp_config_file__(con, conf_num):
 	# Ensure smap existence
 	utils.build_symbol_map()
-	# Load variable count
-	with open(sf(conf.variable_count_file)) as f:
-		f.readline()
-		var_num = int(f.readline())
 	# Write temporally file
 	wfile = tempfile.NamedTemporaryFile(delete=False)
 	for s in con:
@@ -80,7 +76,7 @@ def __write_temp_config_file__(con):
 			s *= -1
 		else:
 			nt = False
-		if s > var_num:
+		if s > conf_num:
 			break;
 		if 'NONAMEGEN' in utils.smap[s]: # ignore generated names
 			continue
@@ -147,10 +143,10 @@ def __calchash__(file):
 	hsh = hashlib.md5(bytes(cstr, 'UTF-8'))
 	return hsh.hexdigest()
 
-def __register_conf__(con):
+def __register_conf__(con, conf_num):
 	dtb = database.database()
 	# Solution to configuration
-	wfile = __write_temp_config_file__(con)
+	wfile = __write_temp_config_file__(con, conf_num)
 	hsh = __calchash__(wfile)
 	filen = os.path.join(sf(conf.configurations_folder), hsh)
 	hshf = hsh
@@ -164,6 +160,32 @@ def __register_conf__(con):
 	shutil.move(wfile, filen)
 	dtb.add_configuration(hsh, hshf)
 
+def __generate_single__(var_num, conf_num):
+	if os.path.isfile(sf(conf.single_generated_file)):
+		return False
+	measure_list = []
+	with open(sf(conf.measure_file), 'r') as f:
+		for ln in f:
+			measure_list.append(int(ln))
+	for measure in measure_list:
+		tfile = __buildtempcnf__(var_num, (sf(conf.rules_file),
+			sf(conf.fixed_file)), (str(measure)))
+		try:
+			confs = __exec_sat__(tfile, ['-i', '0'])
+			for con in confs:
+				__register_conf__(con, conf_num)
+		except exceptions.NoSolution:
+			pass
+		finally:
+			os.remove(tfile)
+	with open(sf(conf.single_generated_file), 'w') as f:
+		f.write("This file informs scripts, that all single selected configurations are already generated.\n")
+		f.write("Remove this file if you want run generating process again.")
+		return True
+
+def __generate_random__(var_num, conf_num):
+	# TODO
+	pass
 
 def generate():
 	"""Collect boolean equations from files rules and required
@@ -172,21 +194,24 @@ def generate():
 	# Check if rules_file exist. If it was generated.
 	if not os.path.isfile(sf(conf.rules_file)):
 		raise exceptions.MissingFile(conf.rules_file,"Run parse_kconfig.")
-	if not os.path.isfile(sf(conf.required_file)):
-		raise exceptions.MissingFile(conf.required_file,"Run allconfig.")
+	if not os.path.isfile(sf(conf.fixed_file)):
+		raise exceptions.MissingFile(conf.required_file,"Run allconfig and initialization process.")
 
-	# Load variable clount
+	# Load variable count
 	with open(sf(conf.variable_count_file)) as f:
 		var_num = f.readline()
-	tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.required_file)), ())
-	try:
-		confs = __exec_sat__(tfile, [])
-		os.remove(tfile)
-		for con in confs:
-			__register_conf__(con)
-	except exceptions.NoSolution:
-		os.remove(tfile)
-		raise exceptions.NoSolution()
+		conf_num = f.readline()
+
+	if __generate_single__(var_num, conf_num):
+		return
+
+	#tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ())
+	#try:
+		#confs = __exec_sat__(tfile, [])
+		#for con in confs:
+			#__register_conf__(con, conf_num)
+	#finally:
+		#os.remove(tfile)
 
 def compare(file1, file2):
 	"""Compared two configuration"""
diff --git a/scripts/exceptions.py b/scripts/exceptions.py
index 730664a..89fba0a 100644
--- a/scripts/exceptions.py
+++ b/scripts/exceptions.py
@@ -14,23 +14,17 @@ class NoSolution(Exception):
 	def __str__(self):
 		return "SAT solver found no solution. Statement is not satisfiable."
 
-class PhaseMismatch(Exception):
-	def __init__(self):
-		pass
-	def __str__(self):
-		return "Phase in " + conf.phase_file + " is unknown."
-
 class ConfigurationError(Exception):
 	def __init__(self, message):
 		self.message = message;
 	def __str__(self):
 		return "Configuration error: " + message
 
-class NoApplicableSolution(Exception):
+class NoApplicableConfiguration(Exception):
 	def __init__(self):
 		pass
 	def __str__(self):
-		return "No applicable solution find. All generated solutions were already applied."
+		return "No applicable configuration find. All generated configurations were already applied."
 
 class ProcessFailed(Exception):
 	def __init__(self, process, returncode):
diff --git a/scripts/initialize.py b/scripts/initialize.py
index f48156d..ee6c43d 100755
--- a/scripts/initialize.py
+++ b/scripts/initialize.py
@@ -9,12 +9,11 @@ import database
 from conf import conf
 from conf import sf
 import exceptions
-import loop
 
 def all():
 	base()
 	parse_kconfig()
-	gen_requred()
+	gen_fixed()
 	# check if database is initialized
 	database.database()
 
@@ -33,17 +32,6 @@ def base():
 	except FileExistsError:
 		pass
 
-	if os.path.isfile(sf(conf.phase_file)):
-		print("Warning: file " + conf.phase_file + " already exists. Not overwritten.")
-	else:
-		loop.phase_set(1)
-
-	if os.path.isfile(sf(conf.iteration_file)):
-		print("Warning: file " + conf.iteration_file + " already exists. Not overwritten.")
-	else:
-		loop.iteration_reset()
-
-
 def parse_kconfig():
 	"Execute parse_kconfig in linux_sources directory."
 	if os.path.isfile(sf(conf.symbol_map_file)) and \
@@ -52,7 +40,6 @@ def parse_kconfig():
 		print('Warning: parse_kconfig not executed. Files already exists.')
 		return
 	print('Executing parse_kconfig...')
-	env = dict(os.environ)
 	wd = os.getcwd()
 	os.chdir(sf(conf.linux_sources))
 	parse_kconfig_cmd = [sf(conf.parse_kconfig)]
@@ -63,8 +50,18 @@ def parse_kconfig():
 	os.chdir(wd)
 
 
-def gen_requred():
-	"Generates required depenpency from dot_config file."
+def __gen_allconfig_fixed__():
+	wd = os.getcwd()
+	os.chdir(sf(conf.linux_sources))
+	allconfig_cmd = [sf(conf.allconfig)]
+	allconfig_cmd += ['Kconfig', sf(conf.dot_config), sf(conf.dot_measure_file)]
+	allconfig_cmd += ['--inv']
+	utils.callsubprocess("allconfig_fixed", allconfig_cmd, False,
+			env = utils.get_kernel_env())
+	os.chdir(wd)
+
+def gen_fixed():
+	"Generates fixed depenpency from dot_config file."
 	print('Generating required configuration...')
 
 	if not os.path.isfile(sf(conf.dot_config)):
@@ -75,9 +72,10 @@ def gen_requred():
 	srmap = {value:key for key, value in utils.smap.items()} # swap dictionary
 
 	shutil.copy(sf(conf.dot_config), sf(conf.dot_config_back_file))
+	__gen_allconfig_fixed__()
 
 	with open(sf(conf.dot_config), 'r') as f:
-		with open(sf(conf.required_file), 'w') as freq:
+		with open(sf(conf.fixed_file), 'w') as ffix:
 			for line in f:
 				if (line[0] == '#') or (not '=' in line):
 					continue
@@ -85,9 +83,18 @@ def gen_requred():
 				if (line[indx + 1] == 'y'):
 					if line[7:indx] == "MODULES": # exception if modules set
 						raise exceptions.ConfigurationError("Fixed kernel configuration must have MODULES disabled.")
-					freq.write(str(srmap[line[7:indx]]) + "\n")
+					ffix.write(str(srmap[line[7:indx]]) + "\n")
 				elif (line[indx + 1] == 'n' or line[indx + 1] == 'm'):
-					freq.write("-" + str(srmap[line[7:indx]]) + "\n")
+					ffix.write("-" + str(srmap[line[7:indx]]) + "\n")
+	with open(sf(conf.dot_measure_file), 'r') as f:
+		with open(sf(conf.measure_file), 'w') as fmes:
+			for line in f:
+				if (line[0] == '#') or (not '=' in line):
+					continue
+				indx = line.index('=')
+				if line[7:indx] == "MODULES":
+					raise exceptions.ConfigurationError("Can't measure configuraion option MODULES. Not supported.")
+				fmes.write(str(srmap[line[7:indx]]) + "\n")
 
 
 #################################################################################
diff --git a/scripts/loop.py b/scripts/loop.py
index fc15d8a..2f008dc 100755
--- a/scripts/loop.py
+++ b/scripts/loop.py
@@ -4,6 +4,7 @@ import sys
 import subprocess
 import signal
 from threading import Thread
+from threading import Lock
 
 from conf import conf
 from conf import sf
@@ -12,145 +13,100 @@ import configurations
 import kernel
 import boot
 import exceptions
+import database
 
-def step():
-	phs = phase_get()
-	if phs == 0 or phs == 1:
-		phase_message(1)
-		initialize.all()
-		phase_set(2)
-	elif phs == 2:
-		phase_message(2)
-		phase_set(3)
-	elif phs == 3:
-		phase_message(3)
-		try:
-			configurations.apply()
-		except exceptions.NoApplicableSolution:
-			try:
-				os.mkdir(sf(conf.result_folder))
-			except FileExistsError:
-				pass
-			print('\nAll done.')
-			exit(0)
-		phase_set(4)
-	elif phs == 4:
-		phase_message(4)
-		phase_set(5)
-	elif phs == 5:
-		phase_message(5)
-		try:
-			kernel.config()
-		except exceptions.ConfigurationError:
-			if not conf.ignore_misconfig:
-				print("Configuration mismatch. Exiting.")
-				sys.exit(-2)
-		phase_set(6)
-	elif phs == 6:
-		phase_message(6)
-		if conf.only_config:
-			phase_set(3)
-		else:
-			phase_set(7)
-	elif phs == 7:
-		phase_message(7)
-		kernel.make()
-		phase_set(8)
-	elif phs == 8:
-		phase_message(8)
-		phase_set(9)
-	elif phs == 9:
-		phase_message(9)
-		boot.boot()
-		phase_set(10)
-	elif phs == 10:
-		phase_message(10)
-		phase_set(3)
+__confs_unmeasured__ = []
 
-# Phase #
-phases = ("Not Initialized",		#0
-		  "Initializing",			#1
-		  "Initialized",			#2
-		  "Solution applying",		#3
-		  "Solution applied",		#4
-		  "Kernel configuration",	#5
-		  "Kernel configured",		#6
-		  "Kernel build",			#7
-		  "Kernel built",			#8
-		  "System boot",			#9
-		  "Benchmark successful"	#10
-		  )
+def prepare():
+	"""Prepare for measuring
+	Outcome is Linux image for generated configuration."""
+	global __confs_unmeasured__
+	if len(__confs_unmeasured__) == 0:
+		dtb = database.database()
+		confs = dtb.get_unmeasured()
+		if len(confs) == 0:
+			configurations.generate()
+			confs = dtb.get_unmeasured()
+			if len(confs) == 0:
+				raise exceptions.NoApplicableConfiguration()
+		__confs_unmeasured__ = list(confs)
+	con = __confs_unmeasured__.pop()
+	kernel.config(con.cfile)
+	img = kernel.make(con.hash)
+	print("Prepared image: " + img)
+	return img, con
 
-def phase_get():
+def measure(kernelimg, con):
 	try:
-		with open(sf(conf.phase_file)) as f:
-			txtPhase = f.readline().rstrip()
-		if not txtPhase in phases:
-			raise PhaseMismatch()
-		return phases.index(txtPhase)
+		os.remove(sf(conf.jobfolder_linux_image))
 	except FileNotFoundError:
-		return 0
-
-def phase_set(phs):
-	# TODO
-	try:
-		global thr
-		if thr.term:
-			return
-	except NameError:
 		pass
-	with open(sf(conf.phase_file), 'w') as f:
-		f.write(phases[phs])
+	os.symlink(os.path.join(sf(conf.build_folder), kernelimg),
+			sf(conf.jobfolder_linux_image))
+	boot.boot(con)
+	print("Configuration '" + con.hash + "' measured.")
 
-def phase_message(phs):
-	"Prints message signaling running phase_"
-	print("-- " + phases[phs])
+# Threads #
+__terminate__ = False
+class mainThread(Thread):
+	def run(self):
+		if conf.single_loop:
+			img, config = prepare()
+			measure(img, config)
+		else:
+			while not __terminate__:
+				img, config = prepare()
+				measure(img, config)
 
-# Iteration #
-def iteration_reset():
-	with open(sf(conf.iteration_file), 'w') as f:
-		f.write('0')
+# Multithread section #
+__conflist__ = []
+__listlock__ = Lock()
 
-def iteration_inc():
-	with open(sf(conf.iteration_file), 'r') as f:
-		it = int(f.readline())
-	it += 1
-	with open(sf(conf.iteration_file), 'w') as f:
-		f.write(str(it))
+class prepareThread(Thread):
+	def __init__(self, name='prepare'):
+		Thread.__init__(self, name=name)
+	def run(self):
+		__listlock__.aquire()
+		while not __terminate__ and len(__conflist__) <= conf.multithread_buffer:
+			__listlock__.release()
+			config = prepare()
+			__listlock__.aquire()
+			__conflist__.append(config)
+			if not __measurethread__.isActive():
+				__measurethread__.start()
+		__listlock__.release()
 
-# Thread #
-class mainThread(Thread):
-	def __init__(self, name):
+class measureThread(Thread):
+	def __init__(self, name='measure'):
 		Thread.__init__(self, name=name)
-		self.term = False
 	def run(self):
-		if conf.step_by_step:
-			step()
-		elif conf.single_loop:
-			while not phase_get() == 2:
-				step()
-			step()
-			while not phase_get() == 2:
-				step()
-		else:
-			while not self.term:
-				step()
+		__listlock__.aquire()
+		while not __terminate__ and len(__conflist__) > 0:
+			config = __conflist__[0]
+			del __conflist__[0]
+			__listlock__.release()
+			if not __preparethread__.isActive():
+				__preparethread__.start()
+			measure(config)
+			__listlock__.aquire()
+		__listlock__.release()
 
-def loop_term():
-	global thr
-	thr.term = True
+__preparethread__ = prepareThread()
+__measurethread__ = measureThread()
 
+# Start and sigterm handler #
 def sigterm_handler(_signo, _stack_frame):
-	loop_term()
+	__terminate__ = True
 
 def loop():
+	initialize.all()
 	global thr
-	thr = mainThread("thred")
+	thr = mainThread()
 	thr.start()
 	try:
 		thr.join()
 	except KeyboardInterrupt:
-		loop_term()
+		__terminate__ = True
 
 #################################################################################
 
diff --git a/scripts/test.py b/scripts/test.py
index 5b94df2..b01a8b6 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -7,15 +7,25 @@ from conf import sf
 import initialize
 import kernel
 import boot
+import database
 
 def test():
 	initialize.base()
 	initialize.parse_kconfig()
-	initialize.gen_requred() # Call this to check initial solution
+	print("-- Make --")
 	conf.kernel_make_output = True
-	kernel.make()
+	img = kernel.make('test')
+	try:
+		os.remove(sf(conf.jobfolder_linux_image))
+	except FileNotFoundError:
+		pass
+	os.symlink(os.path.join(sf(conf.build_folder), img),
+			sf(conf.jobfolder_linux_image))
 	conf.boot_output = True
-	boot.boot()
+	conf.parse_output = True
+	print("-- Boot --")
+	config = database.Config('0', 'test', img)
+	boot.boot(config, False)
 
 #################################################################################
 
diff --git a/scripts/utils.py b/scripts/utils.py
index d138279..7a7a79d 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -29,7 +29,8 @@ def build_symbol_map():
 
 
 def callsubprocess(process_name, process, show_output = True,
-		return_output = False, env=os.environ, allowed_exit_codes = [0]):
+		return_output = False, env=os.environ, allowed_exit_codes = [0],
+		allow_all_exit_codes = False):
 	sprc = subprocess.Popen(process, stdout = subprocess.PIPE, env = env)
 
 	try:
@@ -51,7 +52,7 @@ def callsubprocess(process_name, process, show_output = True,
 				rtn.append(line.rstrip())
 
 	rtncode = sprc.wait()
-	if rtncode not in allowed_exit_codes:
+	if rtncode not in allowed_exit_codes and not allow_all_exit_codes:
 		raise exceptions.ProcessFailed(process, rtncode)
 	return rtn
 
@@ -59,73 +60,3 @@ def get_kernel_env():
 	env = dict(os.environ)
 	env.update(conf.kernel_env)
 	return env
-
-
-def hash_config(cf):
-	"""Hashes configuration using MD5 hash.
-	"""
-	try:
-		cf.remove(0)
-	except ValueError:
-		pass
-	str = ""
-	for c in cf:
-		if c < 0:
-			str += '-'
-		else:
-			str += '+'
-	hsh = hashlib.md5(bytes(str, sys.getdefaultencoding()))
-	return hsh.hexdigest()
-
-def config_strtoint(str, full):
-	"""Reads list of configured symbols from string
-	"""
-	rtn = []
-	if full:
-		for s in str.rstrip().split(sep=' '):
-			rtn.append(int(s))
-	else:
-		count = 0
-		with open(sf(conf.variable_count_file)) as f:
-			f.readline()
-			count = int(f.readline())
-		for s in str.rstrip().split(sep=' '):
-			val = int(s)
-			if abs(val) <= count:
-				rtn.append(val)
-			else:
-				break;
-	try:
-		rtn.remove(0)
-	except ValueError:
-		pass
-	return rtn
-
-def get_config_from_hash(hash):
-	with open(sf(conf.config_map_file), "r") as f:
-		for line in f:
-			w = line.rstrip().split(sep=':')
-			if w[0] == hash:
-				return config_strtoint(w[1], True)
-	return None
-
-def get_last_configuration():
-	hsh = ""
-	try:
-		with open(sf(conf.config_solved_file), "r") as f:
-			for line in f:
-				sline = line.rstrip()
-				if sline != '':
-					hsh = sline
-	except FileNotFoundError:
-		try:
-			with open(sf(conf.config_map_file), "r") as f:
-				w = f.readline().split(sep=':')
-				hsh = w[0]
-		except FileNotFoundError:
-			pass
-
-	if hsh != '':
-		return hsh
-	else:
-		return 'NoConfig'