turris: use force-make machinery to not need to debug on server

This is now configured so that if build of package fails it runs
compilation again with logs printed.

2 files changed, 84 insertions, 5 deletions

diff --git a/scripts/force-make b/scripts/force-make
new file mode 100755
index 0000000..7aa6513
--- /dev/null
+++ b/scripts/force-make
@@ -0,0 +1,82 @@
+#!/bin/bash
+# Make wrapper with ability to decrease paralelism and try again.
+# (C) 2018-2020 CZ.NIC, z.s.p.o.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+set -eu
+
+usage() {
+	echo "Usage: $0 [OPTION].. [-- [ARGUMENTS]..]" >&2
+}
+
+help() {
+	usage
+	cat >&2 <<EOF
+This is make wrapper that tries to run build really hard. It runs build multiple
+times while it decreases paralelism. You can set amount of retries done given
+number of parallel jobs. You can also specify speed how fast number of parallel
+jobs are going to decay using -d option. There is last additional run on top of it
+all that appends all arguments to make call passed to -f option.
+
+Options:
+  -j NUM   Number of parallel jobs, modified and passed to make (in default 1)
+  -d OP    Divider used to decrease number of parallel jobs (in default 2)
+  -c NUM   Number of retries before paralelism is decreased (in default set to 1)
+  -f ARGS  Additional argument to be used for last single job attempt (can be
+           passed multiple times)
+EOF
+}
+
+jobs_count="1"
+jobs_divider="2"
+retry_count="1"
+fallback=()
+while getopts "j:d:c:f:h" opt; do
+	case "$opt" in
+		j)
+			jobs_count="$OPTARG"
+			;;
+		d)
+			jobs_divider="$OPTARG"
+			;;
+		c)
+			retry_count="$OPTARG"
+			;;
+		f)
+			fallback+=("$OPTARG")
+			;;
+		h)
+			help
+			exit 0
+			;;
+		*)
+			usage
+			exit 1
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+
+while [ "$jobs_count" -gt 1 ]; do
+	while [ "$retry_count" -gt 0 ]; do
+		if make -j "$jobs_count" "$@"; then
+			exit 0 # Success
+		fi
+		retry_count="$((retry_count - 1))"
+	done
+	jobs_count="$((jobs_count / jobs_divider))"
+done
+
+make "$@" "${fallback[@]}"
diff --git a/templates/turris.run b/templates/turris.run
index 274e5bc..b0ea26e 100755
--- a/templates/turris.run
+++ b/templates/turris.run
@@ -3,11 +3,8 @@
 . "$(dirname "$(readlink -f "$0")")/../job.common"
 ###################################################
 compile_params="-j$(nproc) -f"
-make="make -j$(($(nproc) * 2)) -l$(nproc) BUILD_LOG=1"
-if [ "$V" -ge 1 ]; then
-	compile_params="-j1 -f"
-	make="make -j1 BUILD_LOG=1 V=99"
-fi
+num_jobs="$(($(nproc) * 2))"
+make="force-make -j$num_jobs -d$num_jobs -f 'V=99' -- -l$(nproc) BUILD_LOG=1"
 ###################################################
 export IS_TTY=1
 # TODO take lock so we won't run multiple instances of new builds