flake-update-20260505
1{
2 globals,
3 lib,
4 pkgs,
5 monitoring,
6 config,
7 ...
8}:
9let
10 # Get machines that should be monitored
11 # Exclude: kyushu (laptop), shikoku (temporarily stopped), nagoya (not yet configured)
12 nodeExporterMachines = lib.filterAttrs (
13 name: _machine:
14 !builtins.elem name [
15 "kyushu"
16 "shikoku"
17 "nagoya"
18 ]
19 ) (monitoring.machinesWithNodeExporter globals.machines);
20
21 # Generate node exporter targets
22 nodeExporterTargets = monitoring.mkPrometheusTargets {
23 machines = nodeExporterMachines;
24 port = 9000;
25 };
26
27 # Machines with BIND DNS
28 bindMachines = lib.filterAttrs (
29 _name: _machine:
30 builtins.elem _name [
31 "demeter"
32 "athena"
33 ]
34 ) globals.machines;
35 bindTargets = monitoring.mkPrometheusTargets {
36 machines = bindMachines;
37 port = 9009;
38 };
39
40 # PostgreSQL hosts
41 postgresTargets = map (host: "${host}.sbr.pm:9187") [
42 "rhea"
43 "sakhalin"
44 ];
45
46 # Exportarr services configuration
47 exportarrServices = {
48 sonarr = {
49 port = 9707;
50 };
51 radarr = {
52 port = 9708;
53 };
54 lidarr = {
55 port = 9709;
56 };
57 prowlarr = {
58 port = 9710;
59 };
60 bazarr = {
61 port = 9712;
62 };
63 };
64 exportarrTargets = lib.mapAttrsToList (
65 _name: cfg: "rhea.sbr.pm:${toString cfg.port}"
66 ) exportarrServices;
67
68 # Docker hosts with metrics enabled
69 dockerMachines = lib.filterAttrs (
70 _name: _machine:
71 builtins.elem _name [
72 "sakhalin"
73 "aomi"
74 ]
75 ) globals.machines;
76 dockerTargets = monitoring.mkPrometheusTargets {
77 machines = dockerMachines;
78 port = 9323;
79 };
80in
81{
82
83 imports = [
84 ../common/services/containers.nix
85 ../common/services/docker.nix
86 ../common/services/binfmt.nix
87
88 ../common/services/prometheus-exporters-postgres.nix
89 ];
90
91 # Disable TPM2 (hardware has no TPM chip)
92 security.tpm2.enable = lib.mkForce false;
93
94 # Age secrets
95 age.secrets."grafana-admin-password" = {
96 file = ../../secrets/sakhalin/grafana-admin-password.age;
97 mode = "400";
98 owner = "grafana";
99 };
100 age.secrets."grafana-secret-key" = {
101 file = ../../secrets/sakhalin/grafana-secret-key.age;
102 mode = "400";
103 owner = "grafana";
104 };
105 age.secrets."ntfy-token" = {
106 file = ../../secrets/sakhalin/ntfy-token.age;
107 mode = "440";
108 owner = "root";
109 group = "root";
110 };
111 age.secrets."homeassistant-prometheus-token" = {
112 file = ../../secrets/sakhalin/homeassistant-prometheus-token.age;
113 mode = "400";
114 owner = "prometheus";
115 };
116 age.secrets."searxng-secret-key" = {
117 file = ../../secrets/sakhalin/searxng-secret-key.age;
118 mode = "400";
119 owner = "searx";
120 group = "searx";
121 };
122
123 systemd.services.n8n.environment = {
124 N8N_SECURE_COOKIE = "false";
125 PATH = lib.mkForce "/run/current-system/sw/bin";
126 };
127
128 services = {
129 atuin = {
130 enable = true;
131 host = "0.0.0.0";
132 openRegistration = false;
133 };
134
135 n8n = {
136 enable = true;
137 openFirewall = true;
138 # webhookUrl = "";
139 };
140 paperless = {
141 enable = false; # Migrated to aion
142 address = "0.0.0.0"; # Listen on all interfaces for access via LAN and VPN
143 port = 8000;
144 dataDir = "/mnt/gaia/paperless/data";
145 mediaDir = "/mnt/gaia/paperless/media";
146 consumptionDir = "/mnt/gaia/paperless/consume";
147 settings = {
148 PAPERLESS_URL = "https://paperless.sbr.pm";
149 PAPERLESS_EMPTY_TRASH_DIR = "/mnt/gaia/paperless/trash";
150 PAPERLESS_FILENAME_FORMAT = "{{ created_year }}/{{ document_type }}/{{ created }} - {{ title }} ({{ doc_pk }})";
151 PAPERLESS_FILENAME_FORMAT_REMOVE_NONE = "true";
152 };
153 };
154 # PostgreSQL backups
155 postgresqlBackup = {
156 enable = true;
157 databases = [ ];
158 location = "/var/backup/postgresql";
159 startAt = "*-*-* 02:15:00"; # Daily at 2:15 AM
160 };
161
162 grafana = {
163 enable = true;
164 settings = {
165 server = {
166 http_addr = "0.0.0.0";
167 http_port = 3000;
168 domain = "grafana.sbr.pm";
169 root_url = "https://grafana.sbr.pm";
170 };
171 security.secret_key = "$__file{${config.age.secrets."grafana-secret-key".path}}";
172 };
173
174 provision = {
175 enable = true;
176 datasources.settings = {
177 apiVersion = 1;
178 datasources = [
179 {
180 name = "Prometheus";
181 type = "prometheus";
182 access = "proxy";
183 url = "http://localhost:9001";
184 isDefault = true;
185 jsonData = {
186 timeInterval = "30s";
187 };
188 }
189 ];
190 };
191
192 dashboards.settings = {
193 apiVersion = 1;
194 providers = [
195 {
196 name = "Default";
197 type = "file";
198 disableDeletion = false;
199 allowUiUpdates = true;
200 options.path = "/var/lib/grafana/dashboards";
201 }
202 ];
203 };
204 };
205 };
206 prometheus = {
207 enable = true;
208 port = 9001;
209 checkConfig = false; # Disable config check due to agenix secrets not available at build time
210
211 # Alert rules
212 ruleFiles = [
213 (pkgs.writeText "prometheus-alerts.yml" (builtins.toJSON (import ./prometheus-alerts.nix)))
214 ];
215
216 # Alertmanager configuration
217 alertmanagers = [
218 {
219 static_configs = [
220 {
221 targets = [ "localhost:9093" ];
222 }
223 ];
224 }
225 ];
226
227 scrapeConfigs = [
228 {
229 job_name = "node";
230 static_configs = [
231 {
232 targets = nodeExporterTargets;
233 }
234 ];
235 }
236 {
237 job_name = "bind";
238 static_configs = [
239 {
240 targets = bindTargets;
241 }
242 ];
243 }
244 {
245 job_name = "postgres";
246 static_configs = [
247 {
248 targets = postgresTargets;
249 }
250 ];
251 }
252 {
253 job_name = "traefik";
254 static_configs = [
255 {
256 targets = [ "rhea.sbr.pm:8080" ];
257 }
258 ];
259 }
260 {
261 job_name = "caddy";
262 static_configs = [
263 {
264 targets = [ "${builtins.head globals.machines.carthage.net.vpn.ips}:2019" ];
265 }
266 ];
267 }
268 {
269 job_name = "exportarr";
270 static_configs = [
271 {
272 targets = exportarrTargets;
273 }
274 ];
275 }
276 # Mosquitto MQTT exporter disabled - package broken in nixpkgs
277 # {
278 # job_name = "mosquitto";
279 # static_configs = [
280 # {
281 # targets = [ "demeter.sbr.pm:9234" ];
282 # }
283 # ];
284 # }
285 {
286 job_name = "homeassistant";
287 static_configs = [
288 {
289 targets = [ "${builtins.head globals.machines.hass.net.ips}:8123" ];
290 }
291 ];
292 metrics_path = "/api/prometheus";
293 bearer_token_file = config.age.secrets."homeassistant-prometheus-token".path;
294 }
295 {
296 job_name = "docker";
297 static_configs = [
298 {
299 targets = dockerTargets;
300 }
301 ];
302 }
303 {
304 job_name = "restic";
305 static_configs = [
306 {
307 targets = [ "aion.sbr.pm:9753" ];
308 }
309 ];
310 }
311 ];
312 };
313
314 # Alertmanager for routing alerts
315 prometheus.alertmanager = {
316 enable = true;
317 port = 9093;
318 webExternalUrl = "http://localhost:9093";
319
320 configuration = {
321 global = {
322 resolve_timeout = "5m";
323 };
324
325 route = {
326 group_by = [
327 "alertname"
328 "instance"
329 ];
330 group_wait = "30s";
331 group_interval = "5m";
332 repeat_interval = "12h";
333 receiver = "ntfy";
334 };
335
336 receivers = [
337 {
338 name = "ntfy";
339 webhook_configs = [
340 {
341 url = "http://localhost:8081/hook"; # alertmanager-ntfy bridge
342 send_resolved = true;
343 }
344 ];
345 }
346 ];
347 };
348 };
349
350 # Local SOCKS5 proxy for SearXNG round-robin (exits through sakhalin's IP)
351 # Paired with carthage's proxy for 50/50 request distribution
352 microsocks = {
353 enable = true;
354 ip = "127.0.0.1";
355 port = 1080;
356 };
357
358 # SearXNG metasearch engine (migrated from aomi)
359 # Private instance, API-focused for Pi agent
360 searx = {
361 enable = true;
362 environmentFile = config.age.secrets."searxng-secret-key".path;
363 settings = {
364 use_default_settings = {
365 engines.remove = [
366 "ahmia"
367 "torch"
368 "startpage"
369 ];
370 };
371 server = {
372 port = 8090;
373 bind_address = "0.0.0.0";
374 secret_key = "$SEARXNG_SECRET_KEY";
375 limiter = false; # Private instance, no rate limiting needed
376 image_proxy = false;
377 };
378 # Route outgoing requests through multiple proxies (round-robin)
379 # to avoid search engine rate limiting / CAPTCHAs
380 outgoing = {
381 request_timeout = 6;
382 retries = 1;
383 proxies = {
384 "all://" = [
385 # sakhalin (local) — exits through sakhalin's IP
386 "socks5h://127.0.0.1:1080"
387 # carthage (Hetzner VPS) — exits through carthage's IP
388 "socks5h://${builtins.head globals.machines.carthage.net.vpn.ips}:1080"
389 ];
390 };
391 extra_proxy_timeout = 10;
392 };
393 search = {
394 safe_search = 0;
395 autocomplete = "";
396 default_lang = "en";
397 formats = [
398 "html"
399 "json"
400 ];
401 # Lower CAPTCHA suspend times so engines recover faster
402 # after proxy rotation provides a fresh IP
403 suspended_times = {
404 SearxEngineCaptcha = 600; # 10min instead of 24h
405 SearxEngineTooManyRequests = 600; # 10min instead of 1h
406 SearxEngineAccessDenied = 1800; # 30min instead of 24h
407 };
408 };
409 # Curated engines for quality results
410 engines = [
411 {
412 name = "duckduckgo";
413 engine = "duckduckgo";
414 shortcut = "ddg";
415 disabled = false;
416 }
417 {
418 name = "google";
419 engine = "google";
420 shortcut = "g";
421 disabled = false;
422 }
423 {
424 name = "brave";
425 engine = "brave";
426 shortcut = "br";
427 disabled = false;
428 }
429 {
430 name = "bing";
431 engine = "bing";
432 shortcut = "bi";
433 disabled = false;
434 }
435 {
436 name = "qwant";
437 engine = "qwant";
438 shortcut = "qw";
439 disabled = false;
440 qwant_categ = "web";
441 }
442 {
443 name = "mojeek";
444 engine = "mojeek";
445 shortcut = "mjk";
446 disabled = false;
447 }
448 {
449 name = "wikipedia";
450 engine = "wikipedia";
451 shortcut = "wp";
452 disabled = false;
453 }
454 {
455 name = "github";
456 engine = "github";
457 shortcut = "gh";
458 disabled = false;
459 }
460 {
461 name = "stackoverflow";
462 engine = "stackexchange";
463 shortcut = "so";
464 disabled = false;
465 categories = "it";
466 }
467 {
468 name = "arch wiki";
469 engine = "archlinux";
470 shortcut = "aw";
471 disabled = false;
472 }
473 {
474 name = "nixos wiki";
475 engine = "mediawiki";
476 shortcut = "nw";
477 disabled = false;
478 base_url = "https://wiki.nixos.org/";
479 search_type = "text";
480 }
481 ];
482 };
483 };
484
485 tarsnap = {
486 enable = true;
487 archives = {
488 documents = {
489 directories = [ "/home/vincent/desktop/documents" ];
490 period = "daily";
491 keyfile = "/etc/nixos/assets/tarsnap.documents.key";
492 };
493 org = {
494 directories = [ "/home/vincent/desktop/org" ];
495 period = "daily";
496 keyfile = "/etc/nixos/assets/tarsnap.org.key";
497 };
498 };
499 };
500 nfs.server = {
501 enable = true;
502 exports = ''
503 /export 192.168.1.0/24(rw,fsid=0,no_subtree_check) 10.100.0.0/24(rw,fsid=0,no_subtree_check)
504 /export/gaia 192.168.1.0/24(rw,fsid=1,no_subtree_check) 10.100.0.0/24(rw,fsid=1,no_subtree_check)
505 /export/toshito 192.168.1.0/24(rw,fsid=2,no_subtree_check) 10.100.0.0/24(rw,fsid=2,no_subtree_check)
506 '';
507 };
508
509 };
510
511 # Create Grafana dashboard directory and deploy Ollama dashboards
512 systemd.tmpfiles.rules = [
513 "d /var/lib/grafana/dashboards 0755 grafana grafana -"
514
515 ];
516
517 # Set Grafana admin password from secret file
518 systemd.services.grafana-set-admin-password = {
519 description = "Set Grafana admin password from secret file";
520 after = [ "grafana.service" ];
521 wantedBy = [ "multi-user.target" ];
522 serviceConfig = {
523 Type = "oneshot";
524 User = "grafana";
525 RemainAfterExit = true;
526 };
527 script = ''
528 # Only set password if admin user exists (database initialized)
529 if ${pkgs.grafana}/bin/grafana-cli --homepath /var/lib/grafana admin reset-admin-password --password-from-stdin < ${
530 config.age.secrets."grafana-admin-password".path
531 } 2>/dev/null; then
532 echo "Admin password updated successfully"
533 else
534 echo "Failed to update password or admin user doesn't exist yet"
535 fi
536 '';
537 };
538
539 # ntfy-alertmanager bridge - manual service configuration with token support
540 systemd.services.alertmanager-ntfy = {
541 description = "Alertmanager to ntfy bridge";
542 after = [ "network.target" ];
543 wantedBy = [ "multi-user.target" ];
544
545 serviceConfig = {
546 Type = "simple";
547 DynamicUser = true;
548 StateDirectory = "alertmanager-ntfy";
549 Restart = "on-failure";
550 RestartSec = "5s";
551 ExecStart = "${pkgs.alertmanager-ntfy}/bin/alertmanager-ntfy --configs /var/lib/alertmanager-ntfy/config.yml";
552 # Run config preparation as root (+ prefix) before starting the main process
553 ExecStartPre =
554 "+"
555 + pkgs.writeShellScript "prepare-alertmanager-ntfy-config" ''
556 # Read the token from the secret file
557 TOKEN=$(cat ${config.age.secrets."ntfy-token".path})
558
559 # Generate config with the actual token
560 cat > /var/lib/alertmanager-ntfy/config.yml <<'EOF'
561 http:
562 addr: 127.0.0.1:8081
563
564 ntfy:
565 baseurl: https://ntfy.sbr.pm
566 auth:
567 token: TOKEN_PLACEHOLDER
568 notification:
569 topic: homelab
570 priority: 'status == "firing" ? "urgent" : "default"'
571 tags:
572 - tag: rotating_light
573 condition: 'status == "firing" && labels.severity == "critical"'
574 - tag: warning
575 condition: 'status == "firing" && labels.severity == "warning"'
576 - tag: "+1"
577 condition: 'status == "resolved"'
578 templates:
579 title: '{{ if eq .Status "resolved" }}✅ Resolved: {{ end }}{{ if eq .Status "firing" }}🔥 {{ end }}{{ index .Annotations "summary" }}'
580 description: '{{ index .Annotations "description" }}'
581 EOF
582 # Replace placeholder with actual token
583 ${pkgs.gnused}/bin/sed -i "s/TOKEN_PLACEHOLDER/$TOKEN/" /var/lib/alertmanager-ntfy/config.yml
584 # Make config readable by the dynamic user
585 chmod 644 /var/lib/alertmanager-ntfy/config.yml
586 '';
587 };
588 };
589
590 environment.systemPackages = with pkgs; [ yt-dlp ];
591 # mr -i u daily
592 systemd.services.mr = {
593 description = "Update configs daily";
594 requires = [ "network-online.target" ];
595 after = [ "network-online.target" ];
596
597 restartIfChanged = false;
598 unitConfig.X-StopOnRemoval = false;
599
600 serviceConfig = {
601 Type = "oneshot";
602 User = "vincent";
603 OnFailure = "status-email-root@%n.service";
604 };
605
606 path = with pkgs; [
607 git
608 mr
609 ];
610 script = ''
611 set -e
612 cd /mnt/gaia/src/configs/
613 mr -t run git reset --hard
614 mr -t u
615 '';
616
617 startAt = "daily";
618 };
619 # Kiwix serve
620 systemd.services.kiwix-serve = {
621 description = "Kiwix offline content server";
622 wantedBy = [ "multi-user.target" ];
623 after = [ "network.target" ];
624
625 serviceConfig = {
626 Type = "simple";
627 User = "vincent";
628 ExecStart = "${pkgs.bash}/bin/bash -c '${pkgs.kiwix-tools}/bin/kiwix-serve --port=8080 /mnt/gaia/kiwix/*.zim'";
629 Restart = "on-failure";
630 RestartSec = "5s";
631 };
632 };
633
634 # Open firewall for services accessible from the network
635 networking.firewall.allowedTCPPorts = [
636 8000 # Paperless-ngx web interface
637 8090 # SearXNG metasearch engine
638 ];
639}