Add monitoring

9 files changed, 163 insertions, 23 deletions

diff --git a/flake.lock b/flake.lock
index 9eefb19..70a03ed 100644
--- a/flake.lock
+++ b/flake.lock
@@ -93,11 +93,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1671015402,
-        "narHash": "sha256-dxslgojapR/YTbKqo71rZ4clTe10f2yyO7+M4nJnWoE=",
+        "lastModified": 1671124523,
+        "narHash": "sha256-YK1wZBsr37DkZVCZvylSShR7DMrW+wKzY/a3vm2mRbA=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "5d77c08858096a3d8f95735ec2227c544f5cdb9c",
+        "rev": "26c7602c390f8c511f326785b570918b2f468892",
         "type": "github"
       },
       "original": {
@@ -126,11 +126,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1670959777,
-        "narHash": "sha256-9nQJWL7S77YZERxairPLFO6TUuF1RgQmdZO6dKRCHz4=",
+        "lastModified": 1671228065,
+        "narHash": "sha256-Az/ig9LVL5xdqtyl4/CVKJIH1G7sP/9Ott2XnNyie0E=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "0fbf27af51a7c9bc68a168fdcd63513c4f100b15",
+        "rev": "e462a4baf75eeac639b4942481759de08a3bc94e",
         "type": "github"
       },
       "original": {
@@ -186,11 +186,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1671005503,
-        "narHash": "sha256-L5pMUoEAxmqwyAivNKvTcNhxL3xY58Zjh3XYtVO2LaQ=",
+        "lastModified": 1671268278,
+        "narHash": "sha256-32gd+9QnaC3HDFJBC5ike9TtbQm1XAwx4ly1Qd0PKNw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f9b0bd5202a0df10856c9fe4cba0074aa0968047",
+        "rev": "97ffa5418460ab0031d2c0e13a4172732a7d0b4a",
         "type": "github"
       },
       "original": {
@@ -266,14 +266,17 @@
     },
     "personal-secret": {
       "locked": {
-        "lastModified": 1670757977,
-        "narHash": "sha256-113ENvU6bc7uz5i2a5mZhEIfhS58WogqusLCE1e90MM=",
+        "lastModified": 1671366822,
+        "narHash": "sha256-6GJP56sj/MhuiYNeWPqIVgDn+V/DEz6Q1CCU8acvQdQ=",
+        "ref": "refs/heads/master",
+        "rev": "268b5854a8a2ecac6c43a00f683019eb0c657d21",
+        "revCount": 51,
         "type": "git",
-        "url": "file:///home/cynerd/projects/admin/nixos-personal-secrets"
+        "url": "ssh://git@cynerd.cz/nixos-personal-secret"
       },
       "original": {
         "type": "git",
-        "url": "file:///home/cynerd/projects/admin/nixos-personal-secrets"
+        "url": "ssh://git@cynerd.cz/nixos-personal-secret"
       }
     },
     "root": {
@@ -297,11 +300,11 @@
         "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1668591389,
-        "narHash": "sha256-s2m5SUU415DRAM9Wr8P2ALT9QLAvNLsyLF8SL49PS7w=",
+        "lastModified": 1671111507,
+        "narHash": "sha256-536GV6hmqMnd/fMijJVwQc2tsJpA1Hq9k0XmDeGPEtY=",
         "ref": "refs/heads/master",
-        "rev": "8c117f39ac6a5b86d8bcfaf39b1238ec8de80274",
-        "revCount": 83,
+        "rev": "78ad772293d320cec8e00b256bec011c577b2553",
+        "revCount": 84,
         "type": "git",
         "url": "https://git.cynerd.cz/shellrc"
       },
@@ -350,11 +353,11 @@
     },
     "vpsadminos": {
       "locked": {
-        "lastModified": 1671009350,
-        "narHash": "sha256-cNKmQdpZsLJmyGhvfAhtDcZf6vb+C50KrFRTH93LT4I=",
+        "lastModified": 1671325230,
+        "narHash": "sha256-jxrkJlrWMXi3+B5d+ROBAUdt4jVcYHa1yQ+5+Bhrx7U=",
         "owner": "vpsfreecz",
         "repo": "vpsadminos",
-        "rev": "9369766af8e8bc32bcadec29ed12dc671c6e6737",
+        "rev": "60e2c4e041de3ae6f40ce5f00a36c77675a26901",
         "type": "github"
       },
       "original": {
diff --git a/hosts.sh b/hosts.sh
index dfd54da..ae108e9 100755
--- a/hosts.sh
+++ b/hosts.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-source "${0%/*}/common.sh"
+source "${0%/*}/tools/common.sh"
 declare -a default_hosts
 ################################################################################
 ## x86_64
diff --git a/local.sh b/local.sh
index 37985c3..809cadd 100755
--- a/local.sh
+++ b/local.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-source "${0%/*}/common.sh"
+source "${0%/*}/tools/common.sh"
 
 operations() {
 	for op in "$@"; do
diff --git a/nixos/modules/default.nix b/nixos/modules/default.nix
index fd60aa4..8bf6a31 100644
--- a/nixos/modules/default.nix
+++ b/nixos/modules/default.nix
@@ -6,6 +6,7 @@ nixpkgs: {
   cynerd-gaming = import ./gaming.nix;
   cynerd-generic = import ./generic.nix;
   cynerd-hosts = import ./hosts.nix;
+  cynerd-monitoring = import ./monitoring.nix;
   cynerd-openvpn = import ./openvpn.nix;
   cynerd-syncthing = import ./syncthing.nix;
   cynerd-wifi-client = import ./wifi-client.nix;
diff --git a/nixos/modules/develop.nix b/nixos/modules/develop.nix
index c1c15b2..fa91d02 100644
--- a/nixos/modules/develop.nix
+++ b/nixos/modules/develop.nix
@@ -59,6 +59,7 @@ in {
         jinja2
         ruamel-yaml
         msgpack
+        urllib3 influxdb-client
 
         psycopg
 
diff --git a/nixos/modules/monitoring.nix b/nixos/modules/monitoring.nix
new file mode 100644
index 0000000..247253e
--- /dev/null
+++ b/nixos/modules/monitoring.nix
@@ -0,0 +1,92 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+cnf = config.cynerd.monitoring;
+hostName = config.networking.hostName;
+isHost = cnf.host == hostName;
+
+in {
+  options.cynerd.monitoring = {
+    enable = mkOption {
+      type = types.bool;
+      default = true;
+      description = "If monitoring should be used";
+    };
+    hw = mkOption {
+      type = types.bool;
+      default = true;
+      description = "If hardware should be reported";
+    };
+
+    host = mkOption {
+      type = types.str;
+      description = "Host name of the monitoring hosting system";
+      readOnly = true;
+    };
+  };
+
+  config = mkMerge [
+    { cynerd.monitoring.host = "ridcully"; }
+    (mkIf cnf.enable {
+      # Telegraf configuration
+      services.telegraf = {
+        enable = true;
+        environmentFiles = ["/run/secrets/telegraf.env"];
+        extraConfig = {
+          agent = {};
+          outputs.influxdb_v2 = {
+            urls = ["http://${cnf.host}:8086"];
+            token = "$INFLUX_TOKEN";
+            organization = "personal";
+            bucket = "monitoring";
+          };
+          inputs = {
+            cpu = {
+              percpu = true;
+              totalcpu = true;
+            };
+            disk = {
+              ignore_fs = [
+                "tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"
+              ];
+            };
+            diskio = {};
+            diskio = {};
+            mem = {};
+            net = {};
+            processes = {};
+            swap = {};
+            system = {};
+          } // (optionalAttrs cnf.hw {
+            sensors = {};
+            smart = {};
+          });
+        };
+      };
+      # TODO probably add this to the upstream configuration
+      systemd.services.telegraf.path = with pkgs; [
+      ] ++ (optionals cnf.hw [
+        nvme-cli lm_sensors smartmontools
+      ]);
+    })
+    (mkIf isHost {
+      # InfluxDB
+      services.influxdb2.enable = mkIf isHost true;
+      # Grafana
+      services.grafana = mkIf isHost {
+        enable = true;
+        settings = {
+          users.allow_sign_up = false;
+          security = {
+            admin_user = "cynerd";
+            admin_password = "$__file{/run/secrets/grafana.admin.pass}";
+          };
+        };
+      };
+
+    })
+  ];
+}
diff --git a/common.sh b/tools/common.sh
index d3ddbc3..d3ddbc3 100644
--- a/common.sh
+++ b/tools/common.sh
diff --git a/tools/influxdb-monitoring.sh b/tools/influxdb-monitoring.sh
new file mode 100755
index 0000000..6488d4f
--- /dev/null
+++ b/tools/influxdb-monitoring.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Generate access tokens for InfluxDB to submit monitoring and other
+# telemetries.
+set -eu
+
+cd "${0%/*}/.."
+
+influx_args=(
+	# Warning: you might want to modify this when you move the InfluxDB host
+	"--host" "http://ridcully:8086"
+	"--token" "$(pass 'nixos-secrets/influxdb/token/cynerd')"
+)
+
+
+monitoring_enabled() {
+	local hostname="$1"
+	[ "$(nix eval ".#nixosConfigurations.$hostname.config.cynerd.monitoring.enable")" = "true" ]
+}
+
+token_is_valid() {
+	[ "$(influx auth list "${influx_args[@]}" --json | jq "map(.token) | any(. == \"$1\")")" = "true" ]
+}
+
+ensure_token() {
+	local hostname="$1"
+	local token
+	pass_path="nixos-secrets/influxdb/token/$hostname"
+	if ! token="$(pass "$pass_path" 2>/dev/null)" \
+		|| ! token_is_valid "$token"; then
+			influx auth create -d "monitoring-$hostname" --write-buckets --json \
+				| jq -r '.token' \
+				| sed 's/^\(.*\)$/\1\n\1/' \
+				| pass insert -f "$pass_path"
+	fi
+}
+
+nix eval --json --apply 'builtins.attrNames' .#nixosConfigurations \
+	| jq -r '.[]' \
+	| while read -r hostname; do
+		if monitoring_enabled "$hostname"; then
+			ensure_token "$hostname"
+		fi
+	done;
diff --git a/install.sh b/tools/install.sh
index f0cc619..71a7aa7 100755
--- a/install.sh
+++ b/tools/install.sh
@@ -2,7 +2,7 @@
 set -eu
 hostname="$1"
 root="${2:-$(pwd)}"
-src="$(readlink -f "${0%/*}")"
+src="$(readlink -f "${0%/*}/..")"
 
 if [ "$(id -u)" -ne 0 ]; then
 	echo "Run this as root!" >&2