Commit f2ab86299ffe

Vincent Demeester <vincent@sbr.pm>
2026-02-23 17:17:47
feat: add carthage (Hetzner Cloud) config
Added NixOS configuration and OpenTofu infra for carthage, the replacement for kerkouane (DigitalOcean). Used disko for declarative partitioning and prepared WireGuard cutover with a single variable switch. Placeholders left for host key, WG pubkey, and syncthing ID until provisioning.
1 parent dd19008
infra/carthage/.gitignore
@@ -0,0 +1,5 @@
+.terraform/
+*.tfstate
+*.tfstate.backup
+*.tfvars
+.terraform.lock.hcl
infra/carthage/main.tf
@@ -0,0 +1,81 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "~> 1.49"
+    }
+  }
+  required_version = ">= 1.6.0"
+}
+
+provider "hcloud" {
+  token = var.hcloud_token
+}
+
+# SSH key for initial access (nixos-anywhere needs this)
+resource "hcloud_ssh_key" "vincent" {
+  name       = "vincent"
+  public_key = file(var.ssh_public_key_path)
+}
+
+# Firewall
+resource "hcloud_firewall" "carthage" {
+  name = "carthage"
+
+  # SSH — for nixos-anywhere bootstrap
+  # Restrict to VPN-only after migration is complete
+  rule {
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "22"
+    source_ips = ["0.0.0.0/0", "::/0"]
+  }
+
+  # HTTP
+  rule {
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "80"
+    source_ips = ["0.0.0.0/0", "::/0"]
+  }
+
+  # HTTPS
+  rule {
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "443"
+    source_ips = ["0.0.0.0/0", "::/0"]
+  }
+
+  # WireGuard
+  rule {
+    direction  = "in"
+    protocol   = "udp"
+    port       = "51820"
+    source_ips = ["0.0.0.0/0", "::/0"]
+  }
+
+  # ICMP (ping)
+  rule {
+    direction  = "in"
+    protocol   = "icmp"
+    source_ips = ["0.0.0.0/0", "::/0"]
+  }
+}
+
+# The server
+resource "hcloud_server" "carthage" {
+  name        = "carthage"
+  server_type = var.server_type
+  location    = var.location
+  image       = "ubuntu-24.04" # Temporary — nixos-anywhere will overwrite
+
+  ssh_keys     = [hcloud_ssh_key.vincent.id]
+  firewall_ids = [hcloud_firewall.carthage.id]
+
+  labels = {
+    environment = "production"
+    role        = "gateway"
+    managed_by  = "opentofu"
+  }
+}
infra/carthage/outputs.tf
@@ -0,0 +1,14 @@
+output "server_ip" {
+  description = "Public IPv4 address of carthage"
+  value       = hcloud_server.carthage.ipv4_address
+}
+
+output "server_ipv6" {
+  description = "Public IPv6 address of carthage"
+  value       = hcloud_server.carthage.ipv6_address
+}
+
+output "server_id" {
+  description = "Hetzner server ID"
+  value       = hcloud_server.carthage.id
+}
infra/carthage/variables.tf
@@ -0,0 +1,23 @@
+variable "hcloud_token" {
+  description = "Hetzner Cloud API token"
+  type        = string
+  sensitive   = true
+}
+
+variable "location" {
+  description = "Hetzner datacenter location"
+  type        = string
+  default     = "fsn1" # Falkenstein, Germany
+}
+
+variable "server_type" {
+  description = "Hetzner server type"
+  type        = string
+  default     = "cx22" # 2 vCPU, 4GB RAM, 40GB NVMe, 20TB traffic
+}
+
+variable "ssh_public_key_path" {
+  description = "Path to SSH public key for initial access"
+  type        = string
+  default     = "~/.ssh/id_ed25519.pub"
+}
lib/default.nix
@@ -84,6 +84,7 @@
         self.nixosModules.gosmee
         self.nixosModules.rsync-replica
         agenixInput.nixosModules.default
+        inputs.disko.nixosModules.disko
         inputs.lanzaboote.nixosModules.lanzaboote
         homeInput.nixosModules.home-manager
         {
systems/carthage/boot.nix
@@ -0,0 +1,39 @@
+{ lib, ... }:
+{
+  console.keyMap = lib.mkForce "us";
+
+  # disko handles GRUB device configuration
+  boot.loader.grub.enable = lib.mkForce true;
+  boot.loader.systemd-boot.enable = lib.mkForce false;
+  boot.initrd.systemd.enable = lib.mkForce false;
+
+  ## Hetzner Cloud (QEMU/KVM guest)
+  boot.initrd.availableKernelModules = [
+    "virtio_net"
+    "virtio_pci"
+    "virtio_mmio"
+    "virtio_blk"
+    "virtio_scsi"
+    "9p"
+    "9pnet_virtio"
+  ];
+  boot.initrd.kernelModules = [
+    "virtio_balloon"
+    "virtio_console"
+    "virtio_rng"
+  ];
+
+  boot.initrd.postDeviceCommands = ''
+    # Set the system time from the hardware clock to work around a
+    # bug in qemu-kvm > 1.5.2 (where the VM clock is initialised
+    # to the *boot time* of the host).
+    hwclock -s
+  '';
+
+  # VPS optimization: No physical hardware, no firmware needed
+  hardware.enableRedistributableFirmware = lib.mkForce false;
+  hardware.enableAllFirmware = lib.mkForce false;
+
+  # VPS optimization: No firmware update service needed
+  services.fwupd.enable = lib.mkForce false;
+}
systems/carthage/extra.nix
@@ -0,0 +1,848 @@
+{
+  config,
+  globals,
+  lib,
+  libx,
+  pkgs,
+  ...
+}:
+let
+  # Common security headers for Caddy
+  securityHeaders = ''
+    header {
+      Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+      X-Content-Type-Options "nosniff"
+      X-Frame-Options "SAMEORIGIN"
+      Referrer-Policy "strict-origin-when-cross-origin"
+      Permissions-Policy "geolocation=(), microphone=(), camera=(), payment=(), usb=(), magnetometer=(), gyroscope=(), accelerometer=()"
+      Content-Security-Policy "default-src 'self' *.sbr.pm *.demeester.fr"
+      X-XSS-Protection "1; mode=block"
+      Cache-Control "public, max-age=604800, immutable"
+      -Server
+    }
+  '';
+
+  # Security headers for media services (more permissive CSP for multimedia)
+  mediaSecurityHeaders = ''
+    header {
+      Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+      X-Content-Type-Options "nosniff"
+      X-Frame-Options "SAMEORIGIN"
+      Referrer-Policy "strict-origin-when-cross-origin"
+      Permissions-Policy "geolocation=(), microphone=(), camera=(), payment=(), usb=()"
+      -Server
+    }
+  '';
+
+  # Security headers for git repository viewer (allow inline scripts/styles for gitmal)
+  gitSecurityHeaders = ''
+    header {
+      Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+      X-Content-Type-Options "nosniff"
+      X-Frame-Options "SAMEORIGIN"
+      Referrer-Policy "strict-origin-when-cross-origin"
+      Permissions-Policy "geolocation=(), microphone=(), camera=(), payment=(), usb=()"
+      Content-Security-Policy "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'"
+      X-XSS-Protection "1; mode=block"
+      -Server
+    }
+  '';
+
+  # Robots.txt snippet - polite request to AI scrapers
+  robotsTxtSnippet = ''
+        @robots path /robots.txt
+        handle @robots {
+          respond 200 {
+            body `User-agent: CCBot
+    Disallow: /
+
+    User-agent: ChatGPT-User
+    Disallow: /
+
+    User-agent: GPTBot
+    Disallow: /
+
+    User-agent: Google-Extended
+    Disallow: /
+
+    User-agent: anthropic-ai
+    Disallow: /
+
+    User-agent: Omgilibot
+    Disallow: /
+
+    User-agent: Omgili
+    Disallow: /
+
+    User-agent: FacebookBot
+    Disallow: /`
+            close
+          }
+        }
+  '';
+
+  # AI bot blocking snippet - enforcement via HTTP 403
+  blockAIBotsSnippet = ''
+    @aibots {
+      header User-Agent *CCBot*
+      header User-Agent *ChatGPT-User*
+      header User-Agent *GPTBot*
+      header User-Agent *Google-Extended*
+      header User-Agent *anthropic-ai*
+      header User-Agent *Omgilibot*
+      header User-Agent *Omgili*
+      header User-Agent *FacebookBot*
+    }
+    handle @aibots {
+      respond "AI scraping not permitted" 403
+    }
+  '';
+in
+{
+  imports = [
+
+    ../common/services/openssh.nix
+  ];
+
+  # ── Fail2ban ────────────────────────────────────────────────────────
+  services.fail2ban = {
+    enable = true;
+
+    # Ban for 1 hour, increase on repeat offenders via recidive jail
+    bantime = "1h";
+    bantime-increment = {
+      enable = true;
+      maxtime = "168h"; # Max 1 week ban
+      factor = "4"; # Aggressive escalation
+    };
+
+    maxretry = 5;
+
+    # Ignore VPN and loopback
+    ignoreIP = [
+      "127.0.0.0/8"
+      "::1"
+      "10.100.0.0/24" # WireGuard VPN
+    ];
+
+    jails = {
+      # Caddy auth failures (401/403 responses)
+      caddy-auth = ''
+        enabled = true
+        backend = auto
+        filter = caddy-auth
+        logpath = /var/log/caddy/access*.log
+        maxretry = 10
+        findtime = 600
+        bantime = 3600
+      '';
+
+      # Caddy aggressive scanning (404 floods)
+      caddy-scan = ''
+        enabled = true
+        backend = auto
+        filter = caddy-scan
+        logpath = /var/log/caddy/access*.log
+        maxretry = 30
+        findtime = 60
+        bantime = 3600
+      '';
+
+      # Caddy rate abuse (too many requests)
+      caddy-flood = ''
+        enabled = true
+        backend = auto
+        filter = caddy-flood
+        logpath = /var/log/caddy/access*.log
+        maxretry = 200
+        findtime = 60
+        bantime = 7200
+      '';
+    };
+  };
+
+  # Caddy fail2ban filters for JSON access logs
+  environment.etc = {
+    # Ban IPs that get too many 401/403 responses (brute force / unauthorized access)
+    "fail2ban/filter.d/caddy-auth.conf".text = ''
+      [Definition]
+      failregex = ^.*"remote_ip":"<HOST>".*"status":(401|403),.*$
+      ignoreregex =
+      datepattern = "ts":{EPOCH}
+    '';
+
+    # Ban IPs that trigger excessive 404s (scanning for vulnerabilities)
+    "fail2ban/filter.d/caddy-scan.conf".text = ''
+      [Definition]
+      failregex = ^.*"remote_ip":"<HOST>".*"status":404,.*$
+      ignoreregex =
+      datepattern = "ts":{EPOCH}
+    '';
+
+    # Ban IPs with excessive request volume (flood / DDoS)
+    "fail2ban/filter.d/caddy-flood.conf".text = ''
+      [Definition]
+      failregex = ^.*"remote_ip":"<HOST>".*"status":\d+,.*$
+      ignoreregex = ^.*"remote_ip":"10\.100\.0\..*$
+      datepattern = "ts":{EPOCH}
+    '';
+  };
+
+  # Age secrets
+  age.secrets."ntfy-token" = {
+    file = ../../secrets/sakhalin/ntfy-token.age;
+    mode = "400";
+    owner = "root";
+    group = "root";
+  };
+
+  # Allow Caddy to access git repositories in vincent's home
+  users.users.caddy.extraGroups = [ "users" ];
+
+  # Allow vincent to run systemd-run without password (for git hooks)
+  # Use /run/current-system/sw/bin path to avoid hardcoded Nix store paths
+  security.sudo.extraRules = [
+    {
+      users = [ "vincent" ];
+      commands = [
+        {
+          command = "/run/current-system/sw/bin/systemd-run";
+          options = [ "NOPASSWD" ];
+        }
+      ];
+    }
+  ];
+
+  # Install gitmal for self-hosted git web view
+  environment.systemPackages = with pkgs; [
+    gitmal
+  ];
+
+  # Git hook background task execution with notifications
+  systemd.services."git-notify@" = {
+    description = "Git build notification for %i";
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${pkgs.writeShellScript "git-notify" ''
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        UNIT_NAME="$1"
+        RESULT=$(${pkgs.systemd}/bin/systemctl show -p Result --value "$UNIT_NAME")
+        EXIT_CODE=$(${pkgs.systemd}/bin/systemctl show -p ExecMainStatus --value "$UNIT_NAME")
+
+        # Get execution timestamps (in microseconds since epoch)
+        START_TIME=$(${pkgs.systemd}/bin/systemctl show -p ExecMainStartTimestamp --value "$UNIT_NAME")
+        EXIT_TIME=$(${pkgs.systemd}/bin/systemctl show -p ExecMainExitTimestamp --value "$UNIT_NAME")
+
+        # Calculate duration in seconds
+        START_EPOCH=$(${pkgs.coreutils}/bin/date -d "$START_TIME" +%s 2>/dev/null || echo "0")
+        EXIT_EPOCH=$(${pkgs.coreutils}/bin/date -d "$EXIT_TIME" +%s 2>/dev/null || echo "0")
+        DURATION=$((EXIT_EPOCH - START_EPOCH))
+
+        # Format duration as human-readable
+        if [ "$DURATION" -ge 60 ]; then
+          MINUTES=$((DURATION / 60))
+          SECONDS=$((DURATION % 60))
+          DURATION_STR="''${MINUTES}m ''${SECONDS}s"
+        else
+          DURATION_STR="''${DURATION}s"
+        fi
+
+        # Parse unit name to extract job type and repo
+        # Format: git-<job>-<repo>-<timestamp>
+        JOB_TYPE=$(echo "$UNIT_NAME" | cut -d'-' -f2)
+        REPO=$(echo "$UNIT_NAME" | cut -d'-' -f3)
+
+        # Only notify on failure
+        if [ "$RESULT" != "success" ]; then
+          ${pkgs.curl}/bin/curl -s \
+            -H "Authorization: Bearer $(${pkgs.coreutils}/bin/tr -d '\n' < ${
+              config.age.secrets."ntfy-token".path
+            })" \
+            -H "Title: ❌ Git $JOB_TYPE Failed: $REPO (after $DURATION_STR)" \
+            -H "Priority: high" \
+            -H "Tags: x,git,$JOB_TYPE,warning" \
+            -d "Job $UNIT_NAME failed after $DURATION_STR (exit code: $EXIT_CODE). Check logs: journalctl -u $UNIT_NAME" \
+            "https://ntfy.sbr.pm/git-builds" || true
+        fi
+      ''} %i";
+    };
+  };
+
+  # Helper script for gitmal generation (called from post-receive hooks)
+  environment.etc."git-hooks/generate-gitmal.sh" = {
+    text = ''
+      #!${pkgs.bash}/bin/bash
+      set -euo pipefail
+
+      # Set PATH to include git and coreutils (gitmal needs git, script needs basename)
+      export PATH="${pkgs.git}/bin:${pkgs.coreutils}/bin:$PATH"
+
+      REPO_PATH="$1"
+      THEME="''${2:-github-dark}"  # Default to 'github-dark' theme if not specified
+      REPO_NAME=$(basename "$REPO_PATH" .git)
+      OUTPUT_DIR="/home/vincent/git/public/$REPO_NAME"
+
+      echo "Generating gitmal for repository: $REPO_NAME"
+      echo "Repository path: $REPO_PATH"
+      echo "Output directory: $OUTPUT_DIR"
+      echo "Theme: $THEME"
+
+      # Generate static site with gitmal
+      cd "$REPO_PATH"
+      ${pkgs.gitmal}/bin/gitmal --output "$OUTPUT_DIR" --theme "$THEME"
+
+      echo "Gitmal generation complete: $OUTPUT_DIR"
+    '';
+    mode = "0755";
+  };
+
+  # Remote build forwarding script
+  environment.etc."git-hooks/forward-build.sh" = {
+    text = ''
+      #!${pkgs.bash}/bin/bash
+      set -euo pipefail
+
+      REPO_NAME="$1"
+      BUILD_TYPE="''${2:-auto}"
+      TIMESTAMP=$(${pkgs.coreutils}/bin/date +%Y%m%d-%H%M%S)
+      UNIT_NAME="build-remote-''${REPO_NAME}-''${TIMESTAMP}"
+
+      echo "Forwarding build to aomi: $REPO_NAME ($BUILD_TYPE)..."
+
+      # SSH to aomi and trigger build with systemd-run
+      ${pkgs.openssh}/bin/ssh -o BatchMode=yes builder@10.100.0.17 \
+        "sudo /run/current-system/sw/bin/systemd-run \
+          --unit=\"$UNIT_NAME\" \
+          --description=\"Remote build: $REPO_NAME ($BUILD_TYPE)\" \
+          --property=\"OnSuccess=job-notify@\''${UNIT_NAME}.service\" \
+          --property=\"OnFailure=job-notify@\''${UNIT_NAME}.service\" \
+          --property=\"User=builder\" \
+          --property=\"Group=users\" \
+          --property=\"WorkingDirectory=/var/lib/git-builds\" \
+          /etc/git-builds/execute-build.sh \"$REPO_NAME\" \"$BUILD_TYPE\""
+
+      echo "✓ Build queued on aomi: $UNIT_NAME"
+      echo "  View status: ssh aomi 'systemctl status $UNIT_NAME'"
+      echo "  View logs:   ssh aomi 'journalctl -u $UNIT_NAME'"
+    '';
+    mode = "0755";
+  };
+
+  # Example post-receive hook template
+  environment.etc."git-hooks/post-receive.example" = {
+    text = ''
+      #!${pkgs.bash}/bin/bash
+      # Example post-receive hook for git repositories
+      # Copy this to your repository's hooks/post-receive and make it executable
+      #
+      # This hook uses systemd-run to execute gitmal generation in the background
+      # with automatic notifications via ntfy when the job completes.
+      #
+      # Optionally, it can also trigger remote builds on aomi.
+
+      set -euo pipefail
+
+      # Configuration
+      GITMAL_ENABLED="true"
+      GITMAL_THEME="github-dark"  # Options: github-dark, github-light, dark, light, auto
+      REMOTE_BUILD_ENABLED="false"  # Set to "true" to enable remote builds on aomi
+      BUILD_TYPE="nixos"             # Options: nixos, make, docker, go, custom, auto
+
+      REPO_PATH="$(pwd)"
+      REPO_NAME=$(basename "$REPO_PATH" .git)
+      TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+
+      # 1. Generate gitmal (local static site on carthage)
+      if [ "$GITMAL_ENABLED" = "true" ]; then
+        UNIT_NAME="git-gitmal-''${REPO_NAME}-''${TIMESTAMP}"
+        echo "Queuing gitmal generation for $REPO_NAME with theme: $GITMAL_THEME..."
+
+        sudo /run/current-system/sw/bin/systemd-run \
+          --unit="$UNIT_NAME" \
+          --description="Gitmal generation for $REPO_NAME" \
+          --property="OnSuccess=git-notify@''${UNIT_NAME}.service" \
+          --property="OnFailure=git-notify@''${UNIT_NAME}.service" \
+          --property="User=vincent" \
+          --property="Group=users" \
+          --working-directory="$REPO_PATH" \
+          /etc/git-hooks/generate-gitmal.sh "$REPO_PATH" "$GITMAL_THEME"
+
+        echo "✓ Gitmal generation queued as: $UNIT_NAME"
+        echo "  View status: systemctl status $UNIT_NAME"
+        echo "  View logs:   journalctl -u $UNIT_NAME"
+      fi
+
+      # 2. Trigger remote build (on aomi)
+      if [ "$REMOTE_BUILD_ENABLED" = "true" ]; then
+        /etc/git-hooks/forward-build.sh "$REPO_NAME" "$BUILD_TYPE"
+      fi
+    '';
+    mode = "0755";
+  };
+
+  # Setup permissions for git directories (via systemd tmpfiles)
+  systemd.tmpfiles.rules = [
+    "d /home/vincent 0711 vincent users -" # Allow traversal to git directory
+    "d /home/vincent/git 0700 vincent users -" # Private git directory
+    "d /home/vincent/git/public 0755 vincent users -" # Public repositories only
+    "d /var/log/git-builds 0755 vincent users -" # Git build logs
+  ];
+
+  # Disable TPM2 (VPS has no TPM hardware)
+  security.tpm2.enable = lib.mkForce false;
+
+  # Override common SSH config to restrict to VPN network only
+  services.openssh = {
+    listenAddresses = [
+      {
+        addr = builtins.head globals.machines.carthage.net.vpn.ips;
+        port = 22;
+      }
+    ];
+    openFirewall = lib.mkForce false;
+  };
+
+  services.wireguard.server = {
+    enable = true;
+    ips = libx.wg-ips globals.machines.carthage.net.vpn.ips;
+    peers = libx.generateWireguardPeers globals.machines;
+  };
+
+  services.gosmee = {
+    enable = true;
+    public-url = "https://webhook.sbr.pm";
+  };
+
+  services.ntfy-sh = {
+    enable = true;
+    settings = {
+      base-url = "https://ntfy.sbr.pm";
+      upstream-base-url = "https://ntfy.sh";
+      listen-http = "localhost:8111";
+      behind-proxy = true;
+      enable-login = true;
+      auth-default-access = "deny-all";
+    };
+  };
+
+  # Firewall configuration
+  # TODO: Migrate to nftables once wireguard server module supports it
+  networking.firewall = {
+    allowPing = true;
+    # Public ports
+    allowedTCPPorts = [
+      80 # HTTP
+      443 # HTTPS
+    ];
+
+    # Additional iptables rules
+    extraCommands = ''
+      # Allow node exporter (9000) only from VPN network
+      iptables -A nixos-fw -p tcp -s 10.100.0.0/24 --dport 9000 -j nixos-fw-accept
+
+      # Block known SYN flood source (USBINF INFORMATICA LTDA, Brazil)
+      iptables -I nixos-fw 1 -s 45.233.176.0/22 -j DROP
+      ip6tables -I nixos-fw 1 -s ::ffff:45.233.176.0/118 -j DROP
+
+      # SYN flood protection: limit new connections per /24 subnet
+      iptables -A nixos-fw -p tcp --syn -m connlimit --connlimit-above 30 --connlimit-mask 24 -j DROP
+      ip6tables -A nixos-fw -p tcp --syn -m connlimit --connlimit-above 30 --connlimit-mask 24 -j DROP
+    '';
+  };
+  # Allow Caddy to access public git repositories only (override ProtectHome)
+  systemd.services.caddy.serviceConfig = {
+    ProtectHome = lib.mkForce "tmpfs"; # Allow read access to /home with bind mounts
+    BindReadOnlyPaths = [ "/home/vincent/git/public" ];
+  };
+
+  services.caddy = {
+    enable = true;
+    email = "vincent@sbr.pm";
+
+    # Use Caddy with rate-limit plugin
+    package = pkgs.caddy.withPlugins {
+      plugins = [ "github.com/mholt/caddy-ratelimit@v0.1.1-0.20250915152450-04ea34edc0c4" ];
+      hash = "sha256-vtiYbCTxotxCQ3+Z+hDOuKy9QxNPKYtVP1FsrtDHq44=";
+    };
+
+    # Enable Prometheus metrics on VPN interface only
+    globalConfig = ''
+      admin ${builtins.head globals.machines.carthage.net.vpn.ips}:2019
+      metrics
+    '';
+
+    # Enable JSON access logging (NixOS option)
+    logFormat = ''
+      output file /var/log/caddy/access.log {
+        roll_size 100MiB
+        roll_keep 10
+        roll_keep_for 720h
+      }
+      format json
+    '';
+
+    virtualHosts = {
+      # File server with directory browsing (replaces fancyindex)
+      "dl.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        root * /var/www/dl.sbr.pm
+        file_server browse {
+          hide .fancyindex README.md HEADER.md
+        }
+
+        ${securityHeaders}
+      '';
+
+      # Alias for dl.sbr.pm
+      "files.sbr.pm".extraConfig = ''
+        redir https://dl.sbr.pm{uri} permanent
+      '';
+
+      # ntfy - reverse proxy with websockets
+      "ntfy.sbr.pm".extraConfig = ''
+        # Rate limiting for notification service
+        rate_limit {
+          zone ntfy_publish {
+            key {remote_host}
+            events 50
+            window 1m
+          }
+        }
+
+        reverse_proxy localhost:8111
+      '';
+
+      # Static sites
+      "paste.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        root * /var/www/paste.sbr.pm
+        file_server
+        ${securityHeaders}
+      '';
+
+      "sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        root * /var/www/sbr.pm
+        file_server
+        ${securityHeaders}
+      '';
+
+      # Go vanity URL service
+      "go.sbr.pm".extraConfig = ''
+        reverse_proxy localhost:8080
+        ${securityHeaders}
+      '';
+
+      # Whoami service (remote)
+      "whoami.sbr.pm".extraConfig = ''
+        reverse_proxy 10.100.0.8:80 {
+          header_up Host {host}
+        }
+      '';
+
+      # Immich photo management (proxied to rhea)
+      "immich.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Allow large photo/video uploads (50GB limit)
+        request_body {
+          max_size 50GB
+        }
+
+        # Strict rate limiting for authentication endpoints
+        @auth {
+          path /auth/* /api/auth/*
+        }
+        route @auth {
+          rate_limit {
+            zone immich_auth {
+              key {remote_host}
+              events 10
+              window 1m
+            }
+          }
+          reverse_proxy 10.100.0.50:2283 {
+            header_up Host {host}
+            header_up X-Real-IP {remote_host}
+          }
+        }
+
+        # Moderate rate limiting for API endpoints
+        @api {
+          path /api/*
+        }
+        route @api {
+          rate_limit {
+            zone immich_api {
+              key {remote_host}
+              events 100
+              window 1m
+            }
+          }
+          reverse_proxy 10.100.0.50:2283 {
+            header_up Host {host}
+            header_up X-Real-IP {remote_host}
+          }
+        }
+
+        # Permissive rate limiting for media/general requests
+        rate_limit {
+          zone immich_media {
+            key {remote_host}
+            events 1000
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.50:2283 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      # Navidrome music streaming (proxied to aion)
+      "navidrome.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Rate limiting for music streaming
+        rate_limit {
+          zone navidrome_general {
+            key {remote_host}
+            events 500
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.49:4533 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      # Jellyfin media server (proxied to rhea)
+      "jellyfin.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Rate limiting for media server
+        rate_limit {
+          zone jellyfin_general {
+            key {remote_host}
+            events 500
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.50:8096 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      # Audiobookshelf audiobook server (proxied to aion)
+      "audiobookshelf.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Rate limiting for audiobook streaming
+        rate_limit {
+          zone audiobookshelf_general {
+            key {remote_host}
+            events 500
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.49:13378 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      # Service aliases (user-friendly URLs - transparent proxy)
+      "music.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Rate limiting for music streaming
+        rate_limit {
+          zone music_general {
+            key {remote_host}
+            events 500
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.49:4533 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      "photos.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Allow large photo/video uploads (50GB limit)
+        request_body {
+          max_size 50GB
+        }
+
+        # Strict rate limiting for authentication endpoints
+        @auth {
+          path /auth/* /api/auth/*
+        }
+        route @auth {
+          rate_limit {
+            zone photos_auth {
+              key {remote_host}
+              events 10
+              window 1m
+            }
+          }
+          reverse_proxy 10.100.0.50:2283 {
+            header_up Host {host}
+            header_up X-Real-IP {remote_host}
+          }
+        }
+
+        # Moderate rate limiting for API endpoints
+        @api {
+          path /api/*
+        }
+        route @api {
+          rate_limit {
+            zone photos_api {
+              key {remote_host}
+              events 100
+              window 1m
+            }
+          }
+          reverse_proxy 10.100.0.50:2283 {
+            header_up Host {host}
+            header_up X-Real-IP {remote_host}
+          }
+        }
+
+        # Permissive rate limiting for media/general requests
+        rate_limit {
+          zone photos_media {
+            key {remote_host}
+            events 1000
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.50:2283 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      "podcasts.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        # Rate limiting for audiobook streaming
+        rate_limit {
+          zone podcasts_general {
+            key {remote_host}
+            events 500
+            window 1m
+          }
+        }
+
+        reverse_proxy 10.100.0.49:13378 {
+          header_up Host {host}
+          header_up X-Real-IP {remote_host}
+        }
+
+        ${mediaSecurityHeaders}
+      '';
+
+      # Webhook/gosmee service with SSE support
+      "webhook.sbr.pm".extraConfig = ''
+        reverse_proxy localhost:3333 {
+          flush_interval -1
+        }
+      '';
+
+      # Personal website with directory browsing
+      "vincent.demeester.fr".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        root * /var/www/vincent.demeester.fr
+
+        # Try files with .html extension
+        try_files {path} {path}.html {path}/ /index.html
+
+        file_server browse {
+          hide .fancyindex README.md HEADER.md
+        }
+
+        ${securityHeaders}
+      '';
+
+      # Self-hosted git repositories (public only)
+      "git.sbr.pm".extraConfig = ''
+        ${blockAIBotsSnippet}
+        ${robotsTxtSnippet}
+
+        root * /home/vincent/git/public
+        file_server browse {
+          hide .fancyindex README.md HEADER.md
+        }
+
+        ${gitSecurityHeaders}
+      '';
+    };
+  };
+
+  services.govanityurl = {
+    enable = true;
+    user = "caddy";
+    host = "go.sbr.pm";
+    config = ''
+      paths:
+        /x:
+          repo: https://github.com/vdemeester/x
+        /lord:
+          repo: https://github.com/vdemeester/lord
+        /ape:
+          repo: https://git.sr.ht/~vdemeester/ape
+        /nr:
+          repo: https://git.sr.ht/~vdemeester/nr
+        /ram:
+          repo: https://git.sr.ht/~vdemeester/ram
+        /sec:
+          repo: https://git.sr.ht/~vdemeester/sec
+    '';
+  };
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "vincent@sbr.pm";
+  };
+}
systems/carthage/hardware.nix
@@ -0,0 +1,48 @@
+{ lib, ... }:
+{
+  # Disko declarative partitioning for Hetzner Cloud
+  # Verify disk device with `lsblk` on the temporary Ubuntu before running nixos-anywhere
+  # Hetzner Cloud typically uses /dev/sda, but confirm first
+  disko.devices = {
+    disk = {
+      main = {
+        device = "/dev/sda";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              size = "1M";
+              type = "EF02"; # BIOS boot partition for GRUB
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+                mountOptions = [
+                  "defaults"
+                  "noatime"
+                ];
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 1024;
+    }
+  ];
+
+  # Hetzner Cloud networking is DHCP-based (simpler than DigitalOcean's static config)
+  networking = {
+    useDHCP = true;
+    usePredictableInterfaceNames = lib.mkForce true;
+  };
+}
systems/carthage/home.nix
@@ -0,0 +1,2 @@
+_: {
+}
systems/common/services/wireguard.nix
@@ -1,5 +1,6 @@
 # Auto-derive WireGuard client config from hostname + globals.
-# Kerkouane (the VPN server) is excluded — it keeps its own server config.
+# VPN servers (kerkouane/carthage) are excluded — they keep their own server config.
+# MIGRATION: Change vpnServer from "kerkouane" to "carthage" during cutover.
 {
   hostname,
   globals,
@@ -8,8 +9,12 @@
   ...
 }:
 let
+  # The active VPN server hostname.
+  # Change to "carthage" when cutting over from DigitalOcean to Hetzner.
+  vpnServer = "kerkouane";
+
   machine = globals.machines.${hostname};
-  isServer = hostname == "kerkouane";
+  isServer = hostname == "kerkouane" || hostname == "carthage";
   hasVpn = machine ? net && machine.net ? vpn;
 in
 {
@@ -18,7 +23,7 @@ in
       enable = true;
       ips = libx.wg-ips machine.net.vpn.ips;
       endpoint = globals.net.vpn.endpoint;
-      endpointPublicKey = globals.machines.kerkouane.net.vpn.pubkey;
+      endpointPublicKey = globals.machines.${vpnServer}.net.vpn.pubkey;
     };
   };
 }
flake.nix
@@ -137,6 +137,12 @@
           homeInput = inputs.home-manager-25_11;
           agenixInput = inputs.agenix-25_11;
         };
+        carthage = libx.mkHost {
+          hostname = "carthage";
+          pkgsInput = inputs.nixpkgs-25_11;
+          homeInput = inputs.home-manager-25_11;
+          agenixInput = inputs.agenix-25_11;
+        };
       };
 
       nixosModules = {
globals.nix
@@ -327,6 +327,29 @@ _: {
         };
       };
     };
+    # Hetzner Cloud VPS — replacement for kerkouane (DigitalOcean)
+    # TODO: Update pubkey, hostKey, and syncthing id after provisioning
+    carthage = {
+      net = {
+        vpn = {
+          pubkey = "PLACEHOLDER_UNTIL_WG_KEYGEN";
+          ips = [ "10.100.0.1" ]; # Takes over kerkouane's VPN server role
+        };
+        names = [
+          "carthage.vpn"
+          "carthage.sbr.pm"
+        ];
+      };
+      ssh = {
+        hostKey = "PLACEHOLDER_UNTIL_INSTALL";
+      };
+      syncthing = {
+        id = "PLACEHOLDER_UNTIL_INSTALL";
+        folders = {
+          sync = { };
+        };
+      };
+    };
     sakhalin = {
       net = {
         ips = [ "192.168.1.70" ];
secrets.nix
@@ -21,6 +21,8 @@ let
   aix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEoUicDySCGETPAgmI0P3UrgZEXXw3zNsyCIylUP0bML"; # ssh-keyscan -q -t ed25519 aix.sbr.pm
   nagoya = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIfep1SkMsAPHggXFLfEJNzZb7eoihtkqDeQruG+TbhF";
   okinawa = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM8vCZ0h6geJZt6i5k6chEDZBggoyq91Z+oNSjvVeSfW"; # From globals.nix
+  # TODO: carthage — add ssh-ed25519 host key after nixos-anywhere install
+  # carthage = "ssh-ed25519 PLACEHOLDER"; # ssh-keyscan -q -t ed25519 carthage.sbr.pm
   # TODO: kobe
   desktops = [
     kyushu