Add implementation of hash indexing of configurations

5 files changed, 174 insertions, 67 deletions

diff --git a/conf.py b/conf.py
index 6340391..829aecd 100644
--- a/conf.py
+++ b/conf.py
@@ -40,6 +40,8 @@ phase_file = build_folder + 'phase'
 symbol_map_file = build_folder + 'symbol_map' # Also defined in parse_kconfig
 rules_file = build_folder + 'rules' # Also defined in parse_kconfig
 variable_count_file = build_folder + 'variable_count' # Also defined in parse_kconfig
+config_map_file = build_folder + 'config_map'
+config_solved_file = build_folder + 'config_solved'
 solved_file = build_folder + 'solved'
 required_file = build_folder + 'required'
 dot_config_fragment_file = build_folder + 'dot_config_fragment'
@@ -49,6 +51,7 @@ iteration_file = build_folder + 'iteration'
 output_confs = build_folder + 'output_confs'
 
 output_folder = 'output/'
+log_folder = 'log/'
 
 nbscript = 'scripts/nbscript'
 
diff --git a/scripts/exceptions.py b/scripts/exceptions.py
index 29aa1dc..ec170ba 100644
--- a/scripts/exceptions.py
+++ b/scripts/exceptions.py
@@ -26,8 +26,16 @@ class ConfigurationError(Exception):
 	def __str__(self):
 		return "Configuration error: " + message
 
-class SolutionGenerated(Exception):
+class NoApplicableSolution(Exception):
 	def __init__(self):
 		pass
 	def __str__(self):
-		return "Solution already generated."
+		return "No applicable solution find. All generated solutions were already applied."
+
+class ProcessFailed(Exception):
+	def __init__(self, process, returncode):
+		self.process = process
+		self.returncode = returncode
+	def __str__(self):
+		return "Process failed: " + str(self.process) + \
+			" with return code: " + str(self.returncode)
diff --git a/scripts/initialize.py b/scripts/initialize.py
index 57654c6..8a1dce8 100755
--- a/scripts/initialize.py
+++ b/scripts/initialize.py
@@ -40,11 +40,11 @@ def parse_kconfig():
 	env = dict(os.environ)
 	wd = os.getcwd()
 	os.chdir(sf(conf.linux_sources))
-	if conf.parse_kconfig_output:
-		subprocess.call([sf(conf.parse_kconfig), sf(conf.linux_kconfig_head), sf(conf.build_folder), "-v", "-v"], env=utils.get_kernel_env())
-	else:
-		subprocess.call([sf(conf.parse_kconfig), sf(conf.linux_kconfig_head), sf(conf.build_folder)], env=utils.get_kernel_env())
-
+	parse_kconfig_cmd = [sf(conf.parse_kconfig)]
+	parse_kconfig_cmd += [sf(conf.linux_kconfig_head), sf(conf.build_folder)]
+	parse_kconfig_cmd += ['-v', '-v']
+	utils.callsubprocess("parse_kconfig", parse_kconfig_cmd,
+			conf.parse_kconfig_output, env=utils.get_kernel_env())
 	os.chdir(wd)
 
 
@@ -58,12 +58,6 @@ def gen_requred():
 	utils.build_symbol_map() # Ensure smap existence
 	srmap = {value:key for key, value in utils.smap.items()}
 
-	try:
-		os.remove(sf(conf.required_file))
-		os.remove(sf(conf.dot_config_fragment_file))
-	except OSError:
-		pass
-
 	shutil.copy(sf(conf.linux_dot_config), sf(conf.dot_config_back_file))
 
 	with open(sf(conf.linux_dot_config), 'r') as f:
@@ -76,12 +70,12 @@ def gen_requred():
 					if (line[7:indx] == "MODULES"): # skip if modules set
 						raise exceptions.ConfigurationError("Initial kernel configuration must have MODULES disabled.")
 					if (line[indx + 1] == 'y'):
-						freq.write(srmap[line[7:indx]] + "\n")
+						freq.write(str(srmap[line[7:indx]]) + "\n")
 					elif (line[indx + 1] == 'n' or line[indx + 1] == 'm'):
-						freq.write("-" + srmap[line[7:indx]] + "\n")
+						freq.write("-" + str(srmap[line[7:indx]]) + "\n")
 					else:
 						fconf.write(line);
-			freq.write("-" + srmap["MODULES"] + "\n"); # force modules no
+			freq.write("-" + str(srmap["MODULES"]) + "\n"); # force modules no
 
 
 def gen_nbscript():
diff --git a/scripts/solution.py b/scripts/solution.py
index 8e02324..f9f605b 100644
--- a/scripts/solution.py
+++ b/scripts/solution.py
@@ -2,6 +2,7 @@ import os
 import sys
 import tempfile
 import subprocess
+import time
 
 import utils
 from conf import conf
@@ -16,7 +17,7 @@ def generate():
 	if not os.path.isfile(sf(conf.rules_file)):
 		raise exceptions.MissingFile(conf.rules_file,"Run parse_kconfig.")
 
-	if sys.path.isfile(sf(conf.solution_file)) and conf.gen_all_solution_oninit:
+	if os.path.isfile(sf(conf.solution_file)) and conf.gen_all_solution_oninit:
 		raise exceptions.SolutionGenerated()
 
 	w_file = tempfile.NamedTemporaryFile(delete=False)
@@ -50,56 +51,69 @@ def generate():
 	w_file.close()
 
 	# Execute picosat
-	picosat_cmd = [conf.picosat, w_file.name]
-	picosat_cmd += ['-o', sf(conf.solution_file)]
+	try:
+		os.mkdir(sf(conf.log_folder))
+	except OSError:
+		pass
+
+	picosat_cmd = [sf(conf.picosat), w_file.name]
 	if (conf.gen_all_solution_oninit):
 		picosat_cmd += ['--all']
-	if conf.picosat_output:
-		subprocess.call(picosat_cmd)
-	else:
-		subprocess.call(picosat_cmd, stdout=subprocess.DEVNULL)
+
+	satprc = subprocess.Popen(picosat_cmd, stdout = subprocess.PIPE)
+	with open(os.path.join(sf(conf.log_folder), "picosat.log"), 'a') as f:
+		f.write("::" + time.strftime("%y-%m-%d-%H-%M-%S") + "::\n")
+		solut = []
+		for linen in satprc.stdout:
+			line = linen.decode(sys.getdefaultencoding())
+			f.write(line)
+			if conf.picosat_output:
+				print(line, end="")
+			if line[0] == 's':
+				if line.rstrip() == 's SATISFIABLE':
+					try:
+						solut.remove(0)
+						with open(sf(conf.config_map_file), 'a') as fm:
+							fm.write(str(utils.hash_config(solut)) + ':')
+							for sl in solut:
+								fm.write(str(sl) + ' ')
+							fm.write('\n')
+						with open(sf(conf.solved_file), 'a') as fs:
+							for sl in solut:
+								fs.write(str(-1 * sl) + ' ')
+							fs.write('\n')
+					except ValueError:
+						pass
+					solut = []
+				else:
+					os.remove(w_file.name)
+					raise exceptions.NoSolution()
+			elif line[0] == 'v':
+				for sl in line[2:].split():
+					solut.append(int(sl))
 
 	os.remove(w_file.name)
 
 def apply():
 	"""Apply generated solution to kernel source.
 	"""
-	# Check if solution_file exist
-	if not os.path.isfile(sf(conf.solution_file)):
-		raise Exception("Solution file is missing. Run sat_solution and check existence of " + sf(conf.solution_file))
-	
 	utils.build_symbol_map() # Ensure smap existence
 
-	# Read solution if satisfiable
-	solut = []
-	with open(sf(conf.solution_file), 'r') as f:
-		if not f.readline().rstrip() == 's SATISFIABLE':
-			raise NoSolution()
-		for line in f:
-			if line[0] == 'v':
-				solut += line[2:].split()
-	solut.remove('0') # Remove 0 at the end 
-
-	# Write solution to output_confs file
-	with open(sf(conf.output_confs), 'a') as f:
-		iteration = 0
-		with open(sf(conf.iteration_file)) as ff:
-			iteration = int(ff.readline())
-		f.write(str(iteration) + ':')
-		for txt in solut:
-			f.write(txt + ' ')
-		f.write('\n')
-
-	# Write negotation solution to solver_file
-	with open(sf(conf.solved_file), 'a') as f:
-		for txt in solut:
-			if txt[0] == '-':
-				ntx = ""
-				txt = txt[1:]
-			else:
-				ntx = "-"
-			f.write( ntx + txt + " ")
-		f.write("\n")
+	solved = set()
+	solution = []
+	if os.path.isfile(sf(config_solved_file)):
+		with open(sf(conf.config_solved_file)) as f:
+			for ln in f:
+				solved.add(ln.strip())
+
+	with open(sf(conf.config_map_file)) as f:
+			while True:
+				w = f.readline().split(sep=':')
+				if not w[0] in solved:
+					solution = utils.config_strtoint(w[1])
+					break;
+	if not solution:
+		raise exceptions.NoApplicableSolution()
 
 	# Load variable count
 	with open(sf(conf.symbol_map_file)) as f:
@@ -108,18 +122,18 @@ def apply():
 		var_num += 1
 	# Write solution to .config file in linux source folder
 	with open(sf(conf.linux_dot_config), 'w') as f:
-		for txt in solut:
-			if txt[0] == '-':
+		for s in solution:
+			if s < 0:
 				nt = True
-				txt = txt[1:]
+				s *= -1
 			else:
 				nt = False
-			if int(txt) >= var_num:
+			if s >= var_num:
 				break;
-			if 'NONAMEGEN' in utils.smap[txt]: # ignore generated names
+			if 'NONAMEGEN' in utils.smap[s]: # ignore generated names
 				continue
 
-			f.write('CONFIG_' + utils.smap[txt] + '=')
+			f.write('CONFIG_' + utils.smap[s] + '=')
 			if not nt:
 				f.write('y')
 			else:
diff --git a/scripts/utils.py b/scripts/utils.py
index 7f8e301..0c10553 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -1,8 +1,12 @@
 import os
 import sys
+import subprocess
+import time
+import hashlib
+import re
 from conf import conf
 from conf import sf
-from exceptions import MissingFile
+import exceptions
 
 def build_symbol_map():
 	"""Generates global variable smap from symbol_map_file.
@@ -14,17 +18,101 @@ def build_symbol_map():
 	except NameError:
 		# Check if symbol_map_file exist
 		if not os.path.isfile(sf(conf.symbol_map_file)):
-			raise MissingFile(sf(conf.symbol_map_file), "Run parse_kconfig to generate it.")
+			raise exceptions.MissingFile(sf(conf.symbol_map_file),
+					"Run parse_kconfig to generate it.")
 
 		smap = dict()
 		with open(sf(conf.symbol_map_file)) as f:
 			for lnn in f:
 				w = lnn.rstrip().split(sep=':')
-				smap[w[0]] = w[1]
-				
+				smap[int(w[0])] = w[1]
+
+
+def build_conf_map():
+	"""Generates global variable cmap from config_map_file and config_solved_file.
+	cmap is dictionary containing list ([configuration], bool solved)
+	cmap is rebuild every time this function is called.
+	"""
+	global cmap
+	cmap = dict()
+	if os.path.isfile(sf(conf.config_map_file)):
+		with open(sf(conf.config_map_file)) as f:
+			for ln in f:
+				w = ln.rstrip().split(sep=':')
+				cf = list()
+				for vr in w[1].split(sep=" "):
+					if vf[0] == '-':
+						cf.append(-1 * int(vf[1:]))
+					cf.append(int(vf))
+				cmap[w[0]] = [w[1], False]
+
+		if os.path.isfile(sf(conf.config_solved_file)):
+			with open(sf(conf.config_solved_file)) as f:
+				for ln in f:
+					try:
+						cmap[ln.rstrip()][1] = True
+					except KeyError:
+						pass
+
+
+def callsubprocess(process_name, process, show_output = True, regular = "",
+		env=os.environ):
+	try:
+		os.mkdir(sf(conf.log_folder))
+	except OSError:
+		pass
+
+	sprc = subprocess.Popen(process, stdout = subprocess.PIPE, env = env)
+
+	rtn = ""
+	with open(os.path.join(sf(conf.log_folder), process_name + ".log"), "a") as f:
+		f.write("::" + time.strftime("%y-%m-%d-%H-%M-%S") + "::\n")
+		for linen in sprc.stdout:
+			line = linen.decode(sys.getdefaultencoding())
+			f.write(line)
+			if show_output:
+				print(line, end="")
+			if re.search(regular, line):
+				rtn += line
+
+	rtncode = sprc.wait()
+	if rtncode != 0:
+		raise exceptions.ProcessFailed(process, rtncode)
+
+	return rtn
+
 def get_kernel_env():
 	env = dict(os.environ)
 	env['SRCARCH'] = conf.SRCARCH
 	env['ARCH'] = conf.ARCH
 	env['KERNELVERSION'] = 'KERNELVERSION' # hides error
 	return env
+
+
+def hash_config(cf):
+	"""Hashes configuration using MD5 hash.
+	"""
+	try:
+		cf.remove(0)
+	except ValueError:
+		pass
+	str = ""
+	for c in cf:
+		if c < 0:
+			str += '-'
+		else:
+			str += '+'
+	hsh = hashlib.md5(bytes(str, sys.getdefaultencoding()))
+	return hsh.hexdigest()
+
+def config_strtoint(str):
+	"""Reads list of configured symbols from string
+	"""
+	rtn = []
+	for s in str.split(sep=' '):
+		rtn.append(int(s))
+	try:
+		rtn.remove(0)
+	except ValueError:
+		pass
+	return rtn