Commit 997e750d70fc

Vincent Demeester <vincent@sbr.pm>
2026-01-05 14:43:35
feat(monitoring): Add Caddy security monitoring and access logging
- Detect potential attacks with 4xx/5xx rate and auth failure alerts - Restrict metrics access to VPN network for security - Enable structured logging for security audit trail Signed-off-by: Vincent Demeester <vincent@sbr.pm>
1 parent b4dfe82
Changed files (2)
systems
systems/kerkouane/extra.nix
@@ -82,12 +82,22 @@ in
     };
   };
 
-  # Should probably move to hardware.nix
-  networking.firewall.allowPing = true;
-  networking.firewall.allowedTCPPorts = [
-    80
-    443
-  ];
+  # Firewall configuration
+  # TODO: Migrate to nftables once wireguard server module supports it
+  networking.firewall = {
+    allowPing = true;
+    # Public ports
+    allowedTCPPorts = [
+      80 # HTTP
+      443 # HTTPS
+    ];
+
+    # Additional iptables rules
+    extraCommands = ''
+      # Allow node exporter (9000) only from VPN network
+      iptables -A nixos-fw -p tcp -s 10.100.0.0/16 --dport 9000 -j nixos-fw-accept
+    '';
+  };
   services.caddy = {
     enable = true;
     email = "vincent@sbr.pm";
@@ -104,6 +114,16 @@ in
       metrics
     '';
 
+    # Enable JSON access logging (NixOS option)
+    logFormat = ''
+      output file /var/log/caddy/access.log {
+        roll_size 100MiB
+        roll_keep 10
+        roll_keep_for 720h
+      }
+      format json
+    '';
+
     virtualHosts = {
       # File server with directory browsing (replaces fancyindex)
       "dl.sbr.pm".extraConfig = ''
systems/sakhalin/prometheus-alerts.nix
@@ -217,6 +217,64 @@
             description = "Caddy reverse proxy has been down for more than 2 minutes - external access may be broken";
           };
         }
+
+        # High 4xx error rate (potential scanning/probing)
+        {
+          alert = "CaddyHigh4xxRate";
+          expr = "rate(caddy_http_requests_total{code=~\"4..\"}[5m]) > 10";
+          for = "5m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High 4xx error rate on Caddy";
+            description = "Host {{ $labels.host }} is seeing {{ $value | humanize }} 4xx errors/sec - potential scanning or misconfiguration";
+          };
+        }
+
+        # High 5xx error rate (server errors)
+        {
+          alert = "CaddyHigh5xxRate";
+          expr = "rate(caddy_http_requests_total{code=~\"5..\"}[5m]) > 1";
+          for = "3m";
+          labels = {
+            severity = "critical";
+          };
+          annotations = {
+            summary = "High 5xx error rate on Caddy";
+            description = "Host {{ $labels.host }} is returning {{ $value | humanize }} 5xx errors/sec - backend services may be failing";
+          };
+        }
+
+        # High request error rate
+        {
+          alert = "CaddyHighErrorRate";
+          expr = "rate(caddy_http_request_errors_total[5m]) > 5";
+          for = "5m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "High request error rate on Caddy";
+            description = "Caddy is encountering {{ $value | humanize }} request errors/sec on {{ $labels.handler }}";
+          };
+        }
+
+        # Suspicious authentication activity (high 401/403 rate on auth endpoints)
+        {
+          alert = "CaddySuspiciousAuthActivity";
+          expr = ''
+            rate(caddy_http_requests_total{code=~"40[13]",host=~"immich.*|jellyfin.*|navidrome.*|audiobookshelf.*"}[5m]) > 5
+          '';
+          for = "3m";
+          labels = {
+            severity = "warning";
+          };
+          annotations = {
+            summary = "Suspicious authentication activity detected";
+            description = "High rate of 401/403 errors on {{ $labels.host }} ({{ $value | humanize }} req/sec) - potential brute force attempt";
+          };
+        }
       ];
     }