Add monitoring

3 files changed, 291 insertions, 0 deletions

diff --git a/tools/common.sh b/tools/common.sh
new file mode 100644
index 0000000..d3ddbc3
--- /dev/null
+++ b/tools/common.sh
@@ -0,0 +1,180 @@
+# Common Bash functions for helper scripts in this repository
+set -eu
+
+## Logging #####################################################################
+_print() {
+	local color="\e[$1m"
+	local clrcolor="\e[0m"
+	shift
+	if [ ! -t 1 ]; then
+		color=""
+		clrcolor=""
+	fi
+	printf "${color}%s${clrcolor}\n" "$*" >&2
+}
+
+stage() {
+	_print '1;32' "$@"
+}
+
+info() {
+	_print '1;35' "$@"
+}
+
+error() {
+	_print '1;31' "$@"
+}
+warning() {
+	_print '1;33' "$@"
+}
+
+## SSH access helper ###########################################################
+
+# Convert hostname to the SSH destination
+sshdest() {
+	if [ "$1" = "lipwig" ]; then
+		echo "newlipwig"
+	else
+		awk -F- 'NF > 1 { print $2"."$1; exit } { print $1 }' <<<"$1"
+	fi
+}
+
+# Reverse opeartion for sshdest
+sshhost() {
+	awk -F. 'NF > 1 { print $2"-"$1; exit } { print $1 }' <<<"$1"
+}
+
+_ssh() {
+	local device="$1"
+	shift
+	if [ "$device" != "$(hostname)" ]; then
+		ssh "$(sshdest "$device")" -- "$@"
+	else
+		if [ $# -gt 1 ]; then
+			"$@"
+		else
+			sh -c "$1"
+		fi
+	fi
+}
+
+_rootssh() {
+	local device="$1"
+	local cmd="$2"
+	if [ "$device" != "$(hostname)" ]; then
+		ssh -t "$(sshdest "$device")" sudo "sh -c '${cmd}'"
+	else
+		sudo sh -c "$cmd"
+	fi
+}
+
+## Evalutions and queries ######################################################
+
+# The path where build result is linked to
+result() {
+	echo ".result-$1"
+}
+
+# Get system of the device
+device_system() {
+	nix eval --raw ".#nixosConfigurations.$1.config.nixpkgs.system"
+}
+
+build_system() {
+	nix eval --raw --impure --expr 'builtins.currentSystem'
+}
+
+# Validates if link is valid.
+build_validate() {
+	local device="$1"
+	[ -L "$(result "$device")" ] && [ -e "$(result "$device")" ]
+}
+
+## Build NixOS system ##########################################################
+# $1: device name
+# All other arguments are passed to the nix build command
+build() {
+	local device="$1"
+	shift
+
+	local toplevel="config.system.build.toplevel"
+	local bsystem="$(build_system)"
+	if [ "$bsystem" != "$(device_system "$device")" ]; then
+		toplevel="config.system.build.cross.$bsystem.$toplevel"
+	fi
+
+	stage "Building system for device: $device"
+	nix build \
+		-o "$(result "${device}")" \
+		--keep-going \
+		"$@" \
+		"${0%/*}#nixosConfigurations.${device}.${toplevel}"
+}
+
+## Copy NixOS system ###########################################################
+# $1: device name
+copy() {
+	local device="$1"
+	if ! build_validate "$device"; then
+		warning "System for device '$device' seems to be not build." >&2
+		return 1
+	fi
+	local store
+	store="$(readlink -f "$(result "$device")")"
+
+	local freespace required;
+	freespace="$(_ssh "$device" df -B 1 /nix | awk 'NR == 2 { print $4 }')"
+	required="$(nix path-info -S "$store" | awk '{ print $2 }')"
+	info "Free space on device: $(numfmt --to=iec "$freespace")"
+	info "Required space: $(numfmt --to=iec "$required")"
+	if [ "$required" -ge "$freespace" ]; then
+		error "There is not enough space to copy clousure to: $device" >&2
+		return 1
+	fi
+
+	stage "Copy closure to: $device"
+	nix copy -s --to "ssh://$(sshdest "$device")" "$store"
+}
+
+## Switch Nix encironment ######################################################
+# $1: switch operation to be performed
+# $2: device name
+# TODO possibly really query if switch is or is not required
+setenv() {
+	local switchop="$1"
+	local device="$2"
+	if ! build_validate "$device"; then
+		warning "System '$device' seems to be not build." >&2
+		return 1
+	fi
+	local store
+	store="$(readlink -f "$(result "$device")")"
+
+	stage "${switchop^} system: $device"
+	local cursystem
+	cursystem="$(_ssh "$device" readlink -f /nix/var/nix/profiles/system)"
+	if [ "$cursystem" != "$store" ]; then
+		info "-----------------------------------------------------------------"
+		_ssh "$device" \
+			nix store diff-closures "$cursystem" "$store"
+		info "-----------------------------------------------------------------"
+		local _store _switchop
+		printf -v _store '%q' "$store"
+		printf -v _switchop '%q' "$switchop"
+		_rootssh "$device" "$_store/bin/nixos-system -s $_switchop"
+	else
+		warning "The latest system might have been already set."
+	fi
+}
+
+boot() {
+	setenv boot "$1"
+}
+
+switch() {
+	setenv switch "$1"
+}
+
+switch_test() {
+	setenv test "$1"
+}
diff --git a/tools/influxdb-monitoring.sh b/tools/influxdb-monitoring.sh
new file mode 100755
index 0000000..6488d4f
--- /dev/null
+++ b/tools/influxdb-monitoring.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Generate access tokens for InfluxDB to submit monitoring and other
+# telemetries.
+set -eu
+
+cd "${0%/*}/.."
+
+influx_args=(
+	# Warning: you might want to modify this when you move the InfluxDB host
+	"--host" "http://ridcully:8086"
+	"--token" "$(pass 'nixos-secrets/influxdb/token/cynerd')"
+)
+
+
+monitoring_enabled() {
+	local hostname="$1"
+	[ "$(nix eval ".#nixosConfigurations.$hostname.config.cynerd.monitoring.enable")" = "true" ]
+}
+
+token_is_valid() {
+	[ "$(influx auth list "${influx_args[@]}" --json | jq "map(.token) | any(. == \"$1\")")" = "true" ]
+}
+
+ensure_token() {
+	local hostname="$1"
+	local token
+	pass_path="nixos-secrets/influxdb/token/$hostname"
+	if ! token="$(pass "$pass_path" 2>/dev/null)" \
+		|| ! token_is_valid "$token"; then
+			influx auth create -d "monitoring-$hostname" --write-buckets --json \
+				| jq -r '.token' \
+				| sed 's/^\(.*\)$/\1\n\1/' \
+				| pass insert -f "$pass_path"
+	fi
+}
+
+nix eval --json --apply 'builtins.attrNames' .#nixosConfigurations \
+	| jq -r '.[]' \
+	| while read -r hostname; do
+		if monitoring_enabled "$hostname"; then
+			ensure_token "$hostname"
+		fi
+	done;
diff --git a/tools/install.sh b/tools/install.sh
new file mode 100755
index 0000000..71a7aa7
--- /dev/null
+++ b/tools/install.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+set -eu
+hostname="$1"
+root="${2:-$(pwd)}"
+src="$(readlink -f "${0%/*}/..")"
+
+if [ "$(id -u)" -ne 0 ]; then
+	echo "Run this as root!" >&2
+	exit 1
+fi
+
+if ! command -v git >/dev/null; then
+	exec nix shell 'nixpkgs#git' -c "$0" "$@"
+fi
+
+if [ ! -s "$root/.personal-secrets.key" ]; then
+	echo "Please paste the personal secret key (terminate using ^D)" >&2
+	sudo tee "$root/.personal-secrets.key" >/dev/null
+fi
+
+if [ -f "$src/flake.nix" ]; then
+	flake="$src"
+else
+	flake="git+https://git.cynerd.cz/nixos-personal"
+	eval "$(ssh-agent)"
+	echo "Please paste the SSH access key now (terminate using ^D):" >&2
+	ssh-add -
+	trap 'kill "$SSH_AGENT_PID"' EXIT
+fi
+
+buildSystem="$(nix eval --raw --impure --expr 'builtins.currentSystem')"
+targetSystem="$(nix eval --raw "$flake#nixosConfigurations.$hostname.pkgs.system")"
+
+toplevel="config.system.build.toplevel"
+if [ "$buildSystem" != "$targetSystem" ]; then
+	toplevel="config.system.build.cross.$buildSystem.$toplevel"
+fi
+
+if [ -f "$src/flake.nix" ]; then
+	# Build in system when running from sources
+	result="$(nix build --no-link --print-out-paths \
+		"$flake#nixosConfigurations.$hostname.$toplevel")"
+	nix copy --no-check-sigs --to "$root" "$result" 
+else
+	result="$(nix build --no-link --print-out-paths \
+		"$flake#nixosConfigurations.$hostname.$toplevel" \
+		--store "$root" --extra-substituters 'auto?trusted=1')"
+fi
+
+nix-env --store "$root" --extra-substituters 'auto?trusted=1' \
+	-p "$root/nix/var/nix/profiles/system" --set "$result"
+
+# Mark the target as a NixOS installation, otherwise switch-to-configuration will chicken out.
+mkdir -m 0755 -p "$root/etc"
+touch "$root/etc/NIXOS"
+
+# Copy over binfmt runners if required
+if [ "$buildSystem" != "$targetSystem" ]; then
+	mkdir -p "$root/run/binfmt"
+	for binfmt in /run/binfmt/*; do
+		nix copy --to "$root" "$(readlink -f "$binfmt")" 
+		ln -sf "$(readlink -f "$binfmt")" "$root/$binfmt"
+	done
+fi
+
+ln -sfn /proc/mounts "$root/etc/mtab" # Grub needs an mtab.
+NIXOS_INSTALL_BOOTLOADER=1 nixos-enter --root "$root" -- \
+	/nix/var/nix/profiles/system/bin/switch-to-configuration boot