Commit 012ee670fa63

Vincent Demeester <vincent@sbr.pm>
2026-04-13 22:41:34
fix: prevent fail2ban from banning home IP
Disabled ntfy-subscriber on kyushu due to expired auth token causing 401 retry floods that triggered caddy-auth fail2ban jail. Added ignoreregex for services with their own auth (immich, navidrome, jellyfin, audiobookshelf, ntfy). Added dynamic home IP whitelist infrastructure via systemd path unit on carthage.
1 parent fe96f9d
Changed files (2)
systems
carthage
kyushu
systems/carthage/extra.nix
@@ -118,7 +118,7 @@ in
 
     maxretry = 5;
 
-    # Ignore VPN and loopback
+    # Ignore VPN, loopback, and dynamic home IP (managed by athena)
     ignoreIP = [
       "127.0.0.0/8"
       "::1"
@@ -164,10 +164,18 @@ in
   # Caddy fail2ban filters for JSON access logs
   environment.etc = {
     # Ban IPs that get too many 401/403 responses (brute force / unauthorized access)
+    # Excludes services with their own auth that naturally return 401 for token refreshes
     "fail2ban/filter.d/caddy-auth.conf".text = ''
       [Definition]
       failregex = ^.*"remote_ip":"<HOST>".*"status":(401|403),.*$
-      ignoreregex =
+      ignoreregex = ^.*"host":"immich\.sbr\.pm".*$
+                    ^.*"host":"photos\.sbr\.pm".*$
+                    ^.*"host":"navidrome\.sbr\.pm".*$
+                    ^.*"host":"music\.sbr\.pm".*$
+                    ^.*"host":"jellyfin\.sbr\.pm".*$
+                    ^.*"host":"audiobookshelf\.sbr\.pm".*$
+                    ^.*"host":"podcasts\.sbr\.pm".*$
+                    ^.*"host":"ntfy\.sbr\.pm".*$
       datepattern = "ts":{EPOCH}
     '';
 
@@ -190,6 +198,82 @@ in
     '';
   };
 
+  # Dynamic home IP whitelist for fail2ban
+  # athena pushes the home public IP to /var/lib/fail2ban/home-ip.txt via SSH
+  # A path unit watches the file and reloads fail2ban ignoreip accordingly
+  systemd.tmpfiles.settings.fail2ban = {
+    "/var/lib/fail2ban".d = {
+      mode = "0755";
+      user = "root";
+      group = "root";
+    };
+  };
+
+  systemd.services.fail2ban-home-ip = {
+    description = "Update fail2ban with dynamic home IP";
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = pkgs.writeShellScript "fail2ban-home-ip" ''
+        #!/usr/bin/env bash
+        set -euo pipefail
+        IP_FILE="/var/lib/fail2ban/home-ip.txt"
+        if [ ! -f "$IP_FILE" ]; then
+          echo "No home IP file found, skipping"
+          exit 0
+        fi
+        NEW_IP=$(${pkgs.coreutils}/bin/cat "$IP_FILE" | ${pkgs.coreutils}/bin/tr -d '[:space:]')
+        if [ -z "$NEW_IP" ]; then
+          echo "Empty IP file, skipping"
+          exit 0
+        fi
+        # Validate IP format
+        if ! echo "$NEW_IP" | ${pkgs.gnugrep}/bin/grep -qP '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'; then
+          echo "Invalid IP format: $NEW_IP"
+          exit 1
+        fi
+        OLD_IP_FILE="/var/lib/fail2ban/home-ip-current.txt"
+        OLD_IP=""
+        if [ -f "$OLD_IP_FILE" ]; then
+          OLD_IP=$(${pkgs.coreutils}/bin/cat "$OLD_IP_FILE" | ${pkgs.coreutils}/bin/tr -d '[:space:]')
+        fi
+        if [ "$NEW_IP" = "$OLD_IP" ]; then
+          echo "Home IP unchanged: $NEW_IP"
+          exit 0
+        fi
+        echo "Updating home IP: $OLD_IP -> $NEW_IP"
+        # Remove old IP from all jails
+        if [ -n "$OLD_IP" ]; then
+          for jail in caddy-auth caddy-flood caddy-scan sshd; do
+            ${pkgs.fail2ban}/bin/fail2ban-client set "$jail" delignoreip "$OLD_IP" 2>/dev/null || true
+            # Also unban in case it was already banned
+            ${pkgs.fail2ban}/bin/fail2ban-client set "$jail" unbanip "$OLD_IP" 2>/dev/null || true
+          done
+        fi
+        # Add new IP to all jails
+        for jail in caddy-auth caddy-flood caddy-scan sshd; do
+          ${pkgs.fail2ban}/bin/fail2ban-client set "$jail" addignoreip "$NEW_IP" 2>/dev/null || true
+          # Unban new IP too in case it was banned before being whitelisted
+          ${pkgs.fail2ban}/bin/fail2ban-client set "$jail" unbanip "$NEW_IP" 2>/dev/null || true
+        done
+        echo "$NEW_IP" > "$OLD_IP_FILE"
+        echo "Home IP updated successfully: $NEW_IP"
+      '';
+    };
+    after = [ "fail2ban.service" ];
+    requires = [ "fail2ban.service" ];
+  };
+
+  systemd.paths.fail2ban-home-ip = {
+    description = "Watch for home IP changes";
+    wantedBy = [ "multi-user.target" ];
+    pathConfig = {
+      PathModified = "/var/lib/fail2ban/home-ip.txt";
+    };
+  };
+
+  # Allow athena to write the home IP file via SSH
+  # athena will: curl -s ifconfig.me | ssh carthage.vpn 'cat > /var/lib/fail2ban/home-ip.txt'
+
   # Age secrets
   age.secrets."ntfy-token" = {
     file = ../../secrets/sakhalin/ntfy-token.age;
systems/kyushu/home.nix
@@ -118,32 +118,33 @@ in
   };
 
   # ntfy notification subscriber
-  systemd.user.services.ntfy-subscriber = {
-    Unit = {
-      Description = "ntfy notification subscriber";
-      Documentation = "https://ntfy.sh";
-      After = [
-        "graphical-session.target"
-        "network-online.target"
-      ];
-      Wants = [ "network-online.target" ];
-    };
-
-    Service = {
-      Type = "simple";
-      ExecStart = "${pkgs.ntfy-sh}/bin/ntfy subscribe --from-config";
-      Restart = "on-failure";
-      RestartSec = 10;
-      Environment = [
-        "PATH=${pkgs.bash}/bin:${pkgs.coreutils}/bin:${pkgs.libnotify}/bin:${pkgs.ntfy-sh}/bin:${pkgs.xdg-utils}/bin:${pkgs.curl}/bin:${pkgs.passage}/bin"
-        "PASSAGE_DIR=/home/vincent/.local/share/passage"
-        "PASSAGE_IDENTITIES_FILE=/home/vincent/.local/share/passage/identities"
-      ];
-    };
-
-    Install = {
-      WantedBy = [ "graphical-session.target" ];
-    };
-  };
+  # disabled: auth token expired, causes fail2ban bans from 401 retry floods
+  # systemd.user.services.ntfy-subscriber = {
+  #   Unit = {
+  #     Description = "ntfy notification subscriber";
+  #     Documentation = "https://ntfy.sh";
+  #     After = [
+  #       "graphical-session.target"
+  #       "network-online.target"
+  #     ];
+  #     Wants = [ "network-online.target" ];
+  #   };
+  #
+  #   Service = {
+  #     Type = "simple";
+  #     ExecStart = "${pkgs.ntfy-sh}/bin/ntfy subscribe --from-config";
+  #     Restart = "on-failure";
+  #     RestartSec = 10;
+  #     Environment = [
+  #       "PATH=${pkgs.bash}/bin:${pkgs.coreutils}/bin:${pkgs.libnotify}/bin:${pkgs.ntfy-sh}/bin:${pkgs.xdg-utils}/bin:${pkgs.curl}/bin:${pkgs.passage}/bin"
+  #       "PASSAGE_DIR=/home/vincent/.local/share/passage"
+  #       "PASSAGE_IDENTITIES_FILE=/home/vincent/.local/share/passage/identities"
+  #     ];
+  #   };
+  #
+  #   Install = {
+  #     WantedBy = [ "graphical-session.target" ];
+  #   };
+  # };
 
 }