Generated configuration is now fully stored to database

Managing configurations in files and in database could cause
inconsistence. Adding all generated configurations to database allow us
to clean project files without loosing data.

5 files changed, 91 insertions, 53 deletions

diff --git a/scripts/configurations.py b/scripts/configurations.py
index 3f2d15e..b346169 100644
--- a/scripts/configurations.py
+++ b/scripts/configurations.py
@@ -65,11 +65,11 @@ def __exec_sat__(file, args):
 		pass
 	return rtn
 
-def __write_temp_config_file__(con, conf_num):
+def __txt_config__(con, conf_num):
 	# Ensure smap existence
 	utils.build_symbol_map()
 	# Write temporally file
-	wfile = tempfile.NamedTemporaryFile(delete=False)
+	txt = ''
 	for s in con:
 		if s < 0:
 			nt = True
@@ -80,41 +80,44 @@ def __write_temp_config_file__(con, conf_num):
 			break;
 		if 'NONAMEGEN' in utils.smap[s]: # ignore generated names
 			continue
-		wfile.write(bytes('CONFIG_' + utils.smap[s] + '=',
-			sys.getdefaultencoding()))
+		txt += 'CONFIG_' + utils.smap[s] + '='
 		if not nt:
-			wfile.write(bytes('y', sys.getdefaultencoding()))
+			txt += 'y'
 		else:
-			wfile.write(bytes('n', sys.getdefaultencoding()))
-		wfile.write(bytes('\n', sys.getdefaultencoding()))
+			txt += 'n'
+		txt += '\n'
+	return txt
+
+def __write_temp_config_file__(con, conf_num):
+	wfile = tempfile.NamedTemporaryFile(delete=False)
+	txt = __txt_config__(con, conf_num)
+	wfile.write(bytes(txt, sys.getdefaultencoding()))
 	wfile.close()
 	return wfile.name
 
-def __load_config_file__(file):
+def __load_config_text__(txt):
 	rtn = dict()
-	with open(file, 'r') as f:
-		for ln in f:
-			if ln[0] == '#' or not '=' in ln:
-				continue
-			indx = ln.index('=')
-			if (ln[indx + 1] == 'y'):
-				rtn[ln[7:indx]] = True
-			else:
-				rtn[ln[7:indx]] = False
+	for ln in txt:
+		if ln[0] == '#' or not '=' in ln:
+			continue
+		indx = ln.index('=')
+		if (ln[indx + 1] == 'y'):
+			rtn[ln[7:indx]] = True
+		else:
+			rtn[ln[7:indx]] = False
 	return rtn
 
-def __calchash__(file):
-	"""Calculates hash from configuration file"""
-	# Build hashconfigsort
-	csort = []
-	try:
-		with open(conf.hashconfigsort, 'r') as f:
-			for ln in f:
-				csort.append(ln.rstrip())
-	except FileNotFoundError:
-		pass
 
-	con = __load_config_file__(file)
+def __load_config_file__(file):
+	f = open(file, 'r')
+	rtn = __load_config_text__(f)
+	f.close()
+	return rtn
+
+def __calchash__(con):
+	dt = database.database()
+	csort = dt.get_configsort()
+
 	cstr = ""
 	for c in csort:
 		try:
@@ -124,38 +127,41 @@ def __calchash__(file):
 			pass
 
 	# Add missing
-	csortfile = open(sf(conf.hashconfigsort), 'a');
 	for key, val in con.items():
 		try:
 			csort.index(key)
 		except ValueError:
 			indx = len(csort)
 			csort.append(key)
-			csortfile.write(key + '\n')
+			dt.add_configsort(key)
 			if val:
 				cstr += key
-	csortfile.close()
 
 	hsh = hashlib.md5(bytes(cstr, 'UTF-8'))
 	return hsh.hexdigest()
 
+
+def __calchash_file__(file):
+	"""Calculates hash from configuration file"""
+	con = __load_config_file__(file)
+	return __calchash__(con)
+
 def __register_conf__(con, conf_num, generator):
 	dtb = database.database()
 	# Solution to configuration
-	wfile = __write_temp_config_file__(con, conf_num)
-	hsh = __calchash__(wfile)
-	filen = os.path.join(sf(conf.configurations_folder), hsh)
-	hshf = hsh
-	if os.path.isfile(filen):
-		if compare(filen, wfile):
+	txtconfig = __txt_config__(con, conf_num)
+	hsh = __calchash__(con)
+	cconf = dtb.get_configration(hsh)
+	for cc in cconf:
+		print('hash: ' + hsh)
+		if compare_text(cc, txtconfig):
 			print("I: Generated existing configuration.")
 			return False
 		else:
 			print("W: Generated configuration with collision hash.")
 			# TODO this might have to be tweaked
 			raise Exception()
-	shutil.move(wfile, filen)
-	dtb.add_configuration(hsh, hshf, generator)
+	dtb.add_configuration(hsh, txtconfig, generator)
 	return True
 
 def __generate_single__(var_num, conf_num):
@@ -218,11 +224,7 @@ def generate():
 
 	raise exceptions.NoNewConfiguration()
 
-def compare(file1, file2):
-	"""Compared two configuration"""
-	conf1 = __load_config_file__(file1)
-	conf2 = __load_config_file__(file2)
-
+def compare(conf1, conf2):
 	# This is not exactly best comparison method
 	for key, val in conf1.items():
 		try:
@@ -237,3 +239,14 @@ def compare(file1, file2):
 		except ValueError:
 			return False
 	return True
+
+def compare_text(text1, text2):
+	conf1 = __load_config_text__(text1)
+	conf2 = __load_config_text__(text2)
+	return compare_file(conf1, conf2)
+
+def compare_file(file1, file2):
+	"""Compared two configuration"""
+	conf1 = __load_config_file__(file1)
+	conf2 = __load_config_file__(file2)
+	return compare_file(conf1, conf2)
diff --git a/scripts/database.py b/scripts/database.py
index 6a4d0a1..c4f0a60 100644
--- a/scripts/database.py
+++ b/scripts/database.py
@@ -19,7 +19,7 @@ def __git_commit__():
 def __timestamp__():
 	return datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S')
 
-Config = collections.namedtuple('Config', 'id hash cfile') # Named tuple for configuration
+Config = collections.namedtuple('Config', 'id hash config') # Named tuple for configuration
 Measure = collections.namedtuple('Measure', 'id conf_id mfile value') # Named tuple for measurement
 
 class database:
@@ -78,21 +78,21 @@ class database:
 		ps(ds, cm)
 		return self.check_linuxgit()
 
-	def add_configuration(self, hash, cfile, generator):
+	def add_configuration(self, hash, txtconfig, generator):
 		"Add configuration to database."
 		ps = self.db.prepare("""INSERT INTO configurations
-								(hash, cfile, gtime, toolgit, linuxgit, generator)
+								(hash, config, gtime, toolgit, linuxgit, generator)
 								VALUES
 								($1, $2, $3, $4, $5, $6);
 								""")
 		gt = self.check_toolsgit()
 		lgt = self.check_linuxgit()
 		tm = datetime.datetime.now()
-		ps(hash, cfile, tm, gt, lgt, generator)
+		ps(hash, txtconfig, tm, gt, lgt, generator)
 
 	def get_configration(self, hash):
 		"Return configration id for inserted hash."
-		ps = self.db.prepare("""SELECT id, cfile FROM configurations
+		ps = self.db.prepare("""SELECT id, config FROM configurations
 								WHERE hash = $1""")
 		rtn = []
 		for dt in ps(hash):
@@ -140,3 +140,17 @@ class database:
 		for dt in ps():
 			rtn.append(Config(dt[0], dt[1], dt[2]))
 		return rtn
+
+	def add_configsort(self, configopt):
+		"Add configuration option to sorted list"
+		ps = self.db.prepare("""INSERT INTO configopt
+								(configopt) VALUES ($1)
+								""")
+		ps()
+
+	def get_configsort(self):
+		"Returns sorted list of all configuration options"
+		ps = self.db.prepare("""SELECT configopt FROM configopt
+								ORDER BY id ASC
+								""")
+		return ps()
diff --git a/scripts/databaseinit.sql b/scripts/databaseinit.sql
index 1e34c75..d1ab3e2 100644
--- a/scripts/databaseinit.sql
+++ b/scripts/databaseinit.sql
@@ -17,7 +17,7 @@ CREATE TABLE configurations (
 	id BIGSERIAL PRIMARY KEY, -- Id
 	hash char(32) NOT NULL, -- Hash of configuration
 	generator TEXT NOT NULL, -- Text identificator of configure generation method
-	cfile TEXT NOT NULL, -- File path with configuration
+	config TEXT NOT NULL, -- Full configuration in text form
 	gtime timestamp NOT NULL, -- Time and date of generation
 	linuxgit BIGINT REFERENCES linuxgit (id), -- Reference to git version of Linux
 	toolgit BIGINT REFERENCES toolsgit (id) -- Reference to git version of tools 
@@ -34,3 +34,10 @@ CREATE TABLE measure (
 	linuxgit BIGINT REFERENCES linuxgit (id), -- Reference to git version of Linux
 	toolgit BIGINT REFERENCES toolsgit (id) -- Reference to git version of tools 
 );
+
+-- In this table are sorted all used configuration options
+-- Order in this table is fundamental for configuration hash calculation
+CREATE TABLE configopt (
+	id BIGSERIAL PRIMARY KEY, -- Id
+	configopt TEXT NOT NULL -- Name of configuration option
+);
diff --git a/scripts/kernel.py b/scripts/kernel.py
index 092de1e..c41fe1f 100644
--- a/scripts/kernel.py
+++ b/scripts/kernel.py
@@ -2,15 +2,18 @@ import os
 import sys
 import subprocess
 import shutil
+import tempfile
 
 from conf import conf
 from conf import sf
 import exceptions
 import utils
 
-def config(cfile):
+def config(txtconfig):
+	"Apply text configuration to kernel folder"
+	infile = tempfile.NamedTemporaryFile()
+	infile.write(bytes(txtconfig, sys.getdefaultencoding()))
 	wd = os.getcwd()
-	infile = os.path.join(sf(conf.configurations_folder), cfile)
 	os.chdir(sf(conf.linux_sources))
 	try:
 		utils.callsubprocess('write_config', [sf(conf.write_config), infile],
diff --git a/scripts/loop.py b/scripts/loop.py
index 9cc8ddb..2a3739e 100755
--- a/scripts/loop.py
+++ b/scripts/loop.py
@@ -20,6 +20,7 @@ __confs_unmeasured__ = []
 def prepare():
 	"""Prepare for measuring
 	Outcome is Linux image for generated configuration."""
+	print("Preparing new image.")
 	global __confs_unmeasured__
 	if len(__confs_unmeasured__) == 0:
 		dtb = database.database()
@@ -31,7 +32,7 @@ def prepare():
 				raise exceptions.NoApplicableConfiguration()
 		__confs_unmeasured__ = list(confs)
 	con = __confs_unmeasured__.pop()
-	kernel.config(con.cfile)
+	kernel.config(con.config)
 	img = kernel.make(con.hash)
 	print("Prepared image: " + img)
 	return img, con