common/headless: basic monitoring on by default

7 files changed, 91 insertions, 37 deletions

diff --git a/.sops.yaml b/.sops.yaml
index ec05e3f..74ce1b2 100644
--- a/.sops.yaml
+++ b/.sops.yaml
@@ -14,3 +14,9 @@ creation_rules:
     - age:
       - *ilex
       - *chaski
+  - path_regex: secrets/common\.yaml$
+    key_groups:
+    - age:
+      - *ilex
+      - *flora
+      - *chaski
diff --git a/common/headless.nix b/common/headless.nix
index 14181ef..da1cdd2 100644
--- a/common/headless.nix
+++ b/common/headless.nix
@@ -1,7 +1,7 @@
 { config, lib, pkgs, ... }:
 
 {
-  imports = [ ./common.nix ];
+  imports = [ ./common.nix ./monitoring.nix ];
 
   # environment.systemPackages = [ pkgs.kitty.terminfo ];
 
diff --git a/common/monitoring.nix b/common/monitoring.nix
new file mode 100644
index 0000000..e5f111c
--- /dev/null
+++ b/common/monitoring.nix
@@ -0,0 +1,40 @@
+{ config, lib, pkgs, ... }:
+
+{
+  # includes mail address, which includes ntfy token
+  sops.secrets."monit/mail".sopsFile = ../secrets/common.yaml;
+
+  services.monit = {
+    enable = true;
+
+    config = ''
+      include /run/secrets/monit/mail
+
+      set daemon 120 with start delay 60
+      set mailserver
+          ping.stuebinm.eu
+          port 2525
+
+      set httpd port 2812 address localhost
+          allow localhost
+
+      check filesystem root with path /
+            if space usage > 80% then alert
+            if inode usage > 80% then alert
+
+      check process sshd with pidfile /var/run/sshd.pid
+            start program  "${pkgs.systemd}/bin/systemctl start sshd"
+            stop program  "${pkgs.systemd}/bin/systemctl stop sshd"
+            if failed port 22 protocol ssh for 2 cycles then restart
+
+      check program is-system-running path ${pkgs.systemd}/bin/systemctl is-system-running
+            if status != 0 then alert
+
+      set mail-format {
+        subject: ${config.networking.hostName}/$SERVICE: $EVENT
+        message: Action: $ACTION $SERVICE: $DESCRIPTION.
+      ($DATE)
+      }
+    '';
+  };
+}
diff --git a/flora/services/monit.nix b/flora/services/monit.nix
index 6c11522..0b7ee7c 100644
--- a/flora/services/monit.nix
+++ b/flora/services/monit.nix
@@ -1,27 +1,9 @@
 { config, lib, pkgs, ... }:
 
 {
-  # includes mail address, which includes ntfy token
-  sops.secrets."monit/mail" = {};
-
   services.monit = {
-    enable = true;
 
     config = ''
-      include /run/secrets/monit/mail
-
-      set daemon 120 with start delay 60
-      set mailserver
-          localhost
-          port 2525
-
-      set httpd port 2812 address localhost
-          allow localhost
-
-      check filesystem root with path /
-            if space usage > 80% then alert
-            if inode usage > 80% then alert
-
       check host stuebinm.eu with address stuebinm.eu
             if failed port 443 with protocol https
             then alert
@@ -34,11 +16,6 @@
             if failed port 64738 of type tcp using ssl with expect "NixOS"
             then alert
 
-      check process sshd with pidfile /var/run/sshd.pid
-            start program  "${pkgs.systemd}/bin/systemctl start sshd"
-            stop program  "${pkgs.systemd}/bin/systemctl stop sshd"
-            if failed port 22 protocol ssh for 2 cycles then restart
-
       check process postfix with pidfile /var/lib/postfix/queue/pid/master.pid
             start program = "${pkgs.systemd}/bin/systemctl start postfix"
             stop program = "${pkgs.systemd}/bin/systemctl stop postfix"
@@ -53,9 +30,6 @@
             if failed port 443 with protocol https
             then alert
 
-      check program is-system-running path ${pkgs.systemd}/bin/systemctl is-system-running
-            if status != 0 then alert
-
       check host hacc-uffd with address login.infra4future.de
             if failed port 443 with protocol https
             then alert
@@ -67,12 +41,6 @@
       check host hacc-mattermost with address mattermost.infra4future.de
             if failed port 443 with protocol https
             then alert
-
-      set mail-format {
-        subject: $SERVICE: $EVENT
-        message: Action: $ACTION $SERVICE: $DESCRIPTION.
-      ($DATE)
-      }
     '';
   };
 
diff --git a/flora/services/ntfy.nix b/flora/services/ntfy.nix
index e00971f..aa46a39 100644
--- a/flora/services/ntfy.nix
+++ b/flora/services/ntfy.nix
@@ -21,6 +21,8 @@
     };
   };
 
+  networking.firewall.allowedTCPPorts = [ 2525 ];
+
   services.mollysocket = {
     enable = true;
     settings = {
diff --git a/secrets/common.yaml b/secrets/common.yaml
new file mode 100644
index 0000000..7fc1a30
--- /dev/null
+++ b/secrets/common.yaml
@@ -0,0 +1,40 @@
+monit:
+    mail: ENC[AES256_GCM,data:QOq9eDdG4r81V1q9N7rCIIjfAhynv7sGMvZs7qdb5tu4qSXp4xcLhE2nk8cJR+XUChi83AC29YH3H7pP17XpguzF,iv:QL6vN7z30QrZDYudmuIg59Kf01TmPZW1UOwh4qWttqc=,tag:SeA5ur9L0cun4RebVRhWCQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age18wkr3kjalalzrq9l05q32gnlaqr7t6rqqzde307m83rs9fp4xcfsdtj9gt
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6akNVendvb1BDeDVhM1ha
+            TEFZdTRqQTBqaGdWeGZhdXJqcHhQcEFDYjA0Clg5SERKM2Y2RFdwdzA0UGZBWkRr
+            ZXN6QWhMcE96bUpoMVhqRjBRbEZVd28KLS0tIDE0UnZsY0RjdHU4UVBMVGhKY3lP
+            N2kwMzFZbmJueXk1Y09USVVEeGJ2QTgKi2bwsKzpo1sZGnUpzOX2gjWxjtEmdnvJ
+            xoVIPzCZfEQeGrtbcwoajFxuZ73ev8aH5x0qBESzyzR+SmaxPZbiiA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1d8hulw7weg6gwxv0cmz969w04d2jkphdx93tm9xs0mqr0ut0t4ls4g4vah
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRYXNzb1VyM2ptaEF6bFls
+            b1g4d25GNnA5QjROZE1xd0hmZTdLOVdCREFvCkRHemdYZS85Y3h3NHRDSTZqcm1s
+            VG5UZWxyMkd5QzBQeWVJMmY1dlIvdWsKLS0tIGM2S0E4QVNVbmdhbG9Na2VDNjI3
+            b0FnaXArbTgvV3cvK0Q4ODFQQkFVRzQKpEny/as9cZ95x/lskW/lKG/vMuncS99t
+            iWlOHpryVFwS9nCLMiOscTSdppnLNbr3FYYoxVfZXeQFgPCYZPlNjQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age14cf8h02c8r2c7nag5fezyhp56za9c4p0t8n39qy452t8hsqwlvgs9y7r8v
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvMStTRDNwdGRIUW1uZXg5
+            amloOUpaSnp2Zm1hZlFrOXlRSkxBZjJDT0NVCklyUHFjaTlTc1Z2QWEwZDBTdkRR
+            YzVORHBnVk9qMVFXVlBMbFN4MFc0L28KLS0tIFdHcXhBUTNOcnFGZGJmMERSU20r
+            dWcwWnpYQnRsTWhzZ3JvYk1XUC9iUzQKApP0h4UxJmCKOOHUN+lkt2dSVCljpP03
+            +0kxmI1ex6aTH9lxQuNBa99OJ2XWZ2/Bmx/sWD7NzcNlRSW/aGMOfg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-04-07T17:49:12Z"
+    mac: ENC[AES256_GCM,data:Veg4lSarFpVaY71tFL0mOR6A/WDxB07O9RXtsxZcukEQqEmay85bJ/C7paoCO+EwU/1isupNA6kWgRGbrV3ts2dUGLQG55MpkK3dRaaADHkV3GThOwvASxer37SznZNvIj1TiVS55UcOl3d+hDI3q5rhQ2RrCHy5dVDvbnG92AM=,iv:jNb8fbNnezWn6Qe717sLGJmUesxR2LAVrQpKpvB5CXA=,tag:/OuZBnDnN4YB5MWJ29usLQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1
diff --git a/secrets/flora.yaml b/secrets/flora.yaml
index 2ea3529..589010a 100644
--- a/secrets/flora.yaml
+++ b/secrets/flora.yaml
@@ -4,8 +4,6 @@ akkoma:
     keyBase: ENC[AES256_GCM,data:E9jPxP8Hg3civkyqHYPdAizisq/Oxw1zHsOmN0XvzPcKlX63ov3Akb1EFGsNqDBoSwTXtMoQk305cMB6VPLqmw==,iv:5c5W83leUmwy3w0dDvkWNdS7JWeseuxEnQc7f98O3bg=,tag:xz5JtAzvqSlkS6FKd8hVhw==,type:str]
     signingSalt: ENC[AES256_GCM,data:/htaDciCAhI=,iv:MV4vYD+qaNBicKZEmYffGfTqE2AQgfUdQVjTrLGPMck=,tag:/Of2A9X2QeE6k4lHwWKcOQ==,type:str]
     jokenDefaultSigner: ENC[AES256_GCM,data:1Wl/N58oiGiGeBHSkJPqLeHOyBmVgLGshAmTyi2H8cu7w/tIHMxW2sd11hhzyq2FCNVsL3Bi+yXgydG7uCl5yw==,iv:criEzJfQMsAUZ7tnIQvr9HOqn7NjBBzXL+rFAgzohPY=,tag:+izDkiUEfwD1+Ym2OuZRnA==,type:str]
-monit:
-    mail: ENC[AES256_GCM,data:wq+xDelBsyIZRJY0GHrZGPWCF0deLZRZxrU89M93hK1zUIeWP6i7xO3dgKE/A5OAGa350Zbj5v9QTieNFHiGqr9g,iv:APUuS3s+t4VPz24Ppen3u+LFSv+GqO49j9Mq77Mb3lQ=,tag:rNVJGN/lnCuq9Km8lZTkLw==,type:str]
 mollysocket:
     config.toml: ENC[AES256_GCM,data:FGT6QOpqaf74yKmUFyyeAPLLv1BEtXZvLrUZw9bCG3hjmd2oUqcX2EGSWWICik3bnfgwYtQAnORg,iv:wEaK5COW9Gm7Hux+Kt8/Md+O/ygSWUk65gMnD6Mnw2g=,tag:4AhZs2vVE2oYErJOC5lMEw==,type:str]
 sops:
@@ -32,8 +30,8 @@ sops:
             SEx0Y2tsaGtkV3dMd0t0ejl3WVkwOW8KTpb14yYJ1bOeLquOrmworNqiwYoZSYiQ
             LkLkXKSGf6T3BrL0t0bM3fgwSQN3k92GGsEZzY7I2hhxZoNXGBOaKg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-03-04T17:19:00Z"
-    mac: ENC[AES256_GCM,data:/GOYEzTEn3fxJRidfPzwgfitcUv2S5MRppiiagH+E1wsEJgV3JtlfxuQ9KQlh1tFPgS1p109+w4udP2dstJGKj027tZT0VJr7KYHFrXzKKdqWypINaqLXOibUg17THHn5W+Y/AFU2hQK1MXem5eY2qCBtxJQMU0ermllY4nuHvA=,iv:KlYG0h6NtPyjrNaLXxpKSO/yQkeW6LqmZl9ZvFNwNdY=,tag:SYrr1grlqOgQcHVJkJzWWQ==,type:str]
+    lastmodified: "2024-04-07T17:49:23Z"
+    mac: ENC[AES256_GCM,data:xPJv1ReXaYtCfvVPXUj+ybPlCE2KkDhF4AAS6nH5l/gy3R5FJsaQZcU784gaqKUcAa74591ENt2Ch7scVlpO3w1y+XHnO51ddgjJ0RXsll3PMaMy3B9dIoxjORBEt7pLNTFPGDkWdbSTPeF9JUypJVvAyC08l0tn3yeCM3oNKEo=,iv:qMk16+6GDb4bqO0ZoV7H1MAOu0K52i3THxOgFd8hYFk=,tag:zMxqN1S60loHfAYKhNn6Cg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1