main
1{
2 globals,
3 lib,
4 pkgs,
5 monitoring,
6 config,
7 ...
8}:
9let
10 # Get machines that should be monitored
11 # Exclude: kyushu (laptop), shikoku (temporarily stopped), nagoya (not yet configured)
12 nodeExporterMachines = lib.filterAttrs (
13 name: _machine:
14 !builtins.elem name [
15 "kyushu"
16 "shikoku"
17 "nagoya"
18 ]
19 ) (monitoring.machinesWithNodeExporter globals.machines);
20
21 # Generate node exporter targets
22 nodeExporterTargets = monitoring.mkPrometheusTargets {
23 machines = nodeExporterMachines;
24 port = 9000;
25 };
26
27 # Machines with BIND DNS
28 bindMachines = lib.filterAttrs (
29 _name: _machine:
30 builtins.elem _name [
31 "demeter"
32 "athena"
33 ]
34 ) globals.machines;
35 bindTargets = monitoring.mkPrometheusTargets {
36 machines = bindMachines;
37 port = 9009;
38 };
39
40 # PostgreSQL hosts
41 postgresTargets = map (host: "${host}.sbr.pm:9187") [
42 "rhea"
43 "sakhalin"
44 ];
45
46 # Exportarr services configuration
47 exportarrServices = {
48 sonarr = {
49 port = 9707;
50 };
51 radarr = {
52 port = 9708;
53 };
54 lidarr = {
55 port = 9709;
56 };
57 prowlarr = {
58 port = 9710;
59 };
60 bazarr = {
61 port = 9712;
62 };
63 };
64 exportarrTargets = lib.mapAttrsToList (
65 _name: cfg: "rhea.sbr.pm:${toString cfg.port}"
66 ) exportarrServices;
67
68 # Docker hosts with metrics enabled
69 dockerMachines = lib.filterAttrs (
70 _name: _machine:
71 builtins.elem _name [
72 "sakhalin"
73 "aomi"
74 ]
75 ) globals.machines;
76 dockerTargets = monitoring.mkPrometheusTargets {
77 machines = dockerMachines;
78 port = 9323;
79 };
80in
81{
82
83 imports = [
84 ../common/services/containers.nix
85 ../common/services/docker.nix
86 ../common/services/binfmt.nix
87
88 ../common/services/prometheus-exporters-postgres.nix
89 ];
90
91 # Disable TPM2 (hardware has no TPM chip)
92 security.tpm2.enable = lib.mkForce false;
93
94 # Age secrets
95 age.secrets."icloud-vdemeester-password" = {
96 file = ../../secrets/mails/icloud-vdemeester.age;
97 mode = "400";
98 owner = "vincent";
99 group = "users";
100 };
101 age.secrets."grafana-admin-password" = {
102 file = ../../secrets/sakhalin/grafana-admin-password.age;
103 mode = "400";
104 owner = "grafana";
105 };
106 age.secrets."grafana-secret-key" = {
107 file = ../../secrets/sakhalin/grafana-secret-key.age;
108 mode = "400";
109 owner = "grafana";
110 };
111 age.secrets."ntfy-token" = {
112 file = ../../secrets/sakhalin/ntfy-token.age;
113 mode = "440";
114 owner = "root";
115 group = "root";
116 };
117 age.secrets."homeassistant-prometheus-token" = {
118 file = ../../secrets/sakhalin/homeassistant-prometheus-token.age;
119 mode = "400";
120 owner = "prometheus";
121 };
122
123 services = {
124 postgresql.package = pkgs.postgresql_16;
125 # PostgreSQL backups
126 postgresqlBackup = {
127 enable = true;
128 databases = [ ];
129 location = "/var/backup/postgresql";
130 startAt = "*-*-* 02:15:00"; # Daily at 2:15 AM
131 };
132
133 grafana = {
134 enable = true;
135 settings = {
136 server = {
137 http_addr = "0.0.0.0";
138 http_port = 3000;
139 domain = "grafana.sbr.pm";
140 root_url = "https://grafana.sbr.pm";
141 };
142 security.secret_key = "$__file{${config.age.secrets."grafana-secret-key".path}}";
143 };
144
145 provision = {
146 enable = true;
147 datasources.settings = {
148 apiVersion = 1;
149 datasources = [
150 {
151 name = "Prometheus";
152 type = "prometheus";
153 access = "proxy";
154 url = "http://localhost:9001";
155 isDefault = true;
156 jsonData = {
157 timeInterval = "30s";
158 };
159 }
160 ];
161 };
162
163 dashboards.settings = {
164 apiVersion = 1;
165 providers = [
166 {
167 name = "Default";
168 type = "file";
169 disableDeletion = false;
170 allowUiUpdates = true;
171 options.path = "/var/lib/grafana/dashboards";
172 }
173 ];
174 };
175 };
176 };
177 prometheus = {
178 enable = true;
179 port = 9001;
180 checkConfig = false; # Disable config check due to agenix secrets not available at build time
181
182 # Alert rules
183 ruleFiles = [
184 (pkgs.writeText "prometheus-alerts.yml" (builtins.toJSON (import ./prometheus-alerts.nix)))
185 ];
186
187 # Alertmanager configuration
188 alertmanagers = [
189 {
190 static_configs = [
191 {
192 targets = [ "localhost:9093" ];
193 }
194 ];
195 }
196 ];
197
198 scrapeConfigs = [
199 {
200 job_name = "node";
201 static_configs = [
202 {
203 targets = nodeExporterTargets;
204 }
205 ];
206 }
207 {
208 job_name = "bind";
209 static_configs = [
210 {
211 targets = bindTargets;
212 }
213 ];
214 }
215 {
216 job_name = "postgres";
217 static_configs = [
218 {
219 targets = postgresTargets;
220 }
221 ];
222 }
223 {
224 job_name = "traefik";
225 static_configs = [
226 {
227 targets = [ "rhea.sbr.pm:8080" ];
228 }
229 ];
230 }
231 {
232 job_name = "caddy";
233 static_configs = [
234 {
235 targets = [ "${builtins.head globals.machines.carthage.net.vpn.ips}:2019" ];
236 }
237 ];
238 }
239 {
240 job_name = "exportarr";
241 static_configs = [
242 {
243 targets = exportarrTargets;
244 }
245 ];
246 }
247 # Mosquitto MQTT exporter disabled - package broken in nixpkgs
248 # {
249 # job_name = "mosquitto";
250 # static_configs = [
251 # {
252 # targets = [ "demeter.sbr.pm:9234" ];
253 # }
254 # ];
255 # }
256 {
257 job_name = "homeassistant";
258 static_configs = [
259 {
260 targets = [ "${builtins.head globals.machines.hass.net.ips}:8123" ];
261 }
262 ];
263 metrics_path = "/api/prometheus";
264 bearer_token_file = config.age.secrets."homeassistant-prometheus-token".path;
265 }
266 {
267 job_name = "docker";
268 static_configs = [
269 {
270 targets = dockerTargets;
271 }
272 ];
273 }
274 {
275 job_name = "restic";
276 static_configs = [
277 {
278 targets = [ "aion.sbr.pm:9753" ];
279 }
280 ];
281 }
282 ];
283 };
284
285 # Alertmanager for routing alerts
286 prometheus.alertmanager = {
287 enable = true;
288 port = 9093;
289 webExternalUrl = "http://localhost:9093";
290
291 configuration = {
292 global = {
293 resolve_timeout = "5m";
294 };
295
296 route = {
297 group_by = [
298 "alertname"
299 "instance"
300 ];
301 group_wait = "30s";
302 group_interval = "5m";
303 repeat_interval = "12h";
304 receiver = "ntfy";
305 };
306
307 receivers = [
308 {
309 name = "ntfy";
310 webhook_configs = [
311 {
312 url = "http://localhost:8081/hook"; # alertmanager-ntfy bridge
313 send_resolved = true;
314 }
315 ];
316 }
317 ];
318 };
319 };
320
321 nfs.server = {
322 enable = true;
323 exports = ''
324 /export 192.168.1.0/24(rw,fsid=0,no_subtree_check) 10.100.0.0/24(rw,fsid=0,no_subtree_check)
325 /export/gaia 192.168.1.0/24(rw,fsid=1,no_subtree_check) 10.100.0.0/24(rw,fsid=1,no_subtree_check)
326 /export/toshito 192.168.1.0/24(rw,fsid=2,no_subtree_check) 10.100.0.0/24(rw,fsid=2,no_subtree_check)
327 '';
328 };
329
330 };
331
332 # Create Grafana dashboard directory and deploy Ollama dashboards
333 systemd.tmpfiles.rules = [
334 "d /var/lib/grafana/dashboards 0755 grafana grafana -"
335
336 ];
337
338 # Set Grafana admin password from secret file
339 systemd.services.grafana-set-admin-password = {
340 description = "Set Grafana admin password from secret file";
341 after = [ "grafana.service" ];
342 wantedBy = [ "multi-user.target" ];
343 serviceConfig = {
344 Type = "oneshot";
345 User = "grafana";
346 RemainAfterExit = true;
347 };
348 script = ''
349 # Only set password if admin user exists (database initialized)
350 if ${pkgs.grafana}/bin/grafana-cli --homepath /var/lib/grafana admin reset-admin-password --password-from-stdin < ${
351 config.age.secrets."grafana-admin-password".path
352 } 2>/dev/null; then
353 echo "Admin password updated successfully"
354 else
355 echo "Failed to update password or admin user doesn't exist yet"
356 fi
357 '';
358 };
359
360 # ntfy-alertmanager bridge - manual service configuration with token support
361 systemd.services.alertmanager-ntfy = {
362 description = "Alertmanager to ntfy bridge";
363 after = [ "network.target" ];
364 wantedBy = [ "multi-user.target" ];
365
366 serviceConfig = {
367 Type = "simple";
368 DynamicUser = true;
369 StateDirectory = "alertmanager-ntfy";
370 Restart = "on-failure";
371 RestartSec = "5s";
372 ExecStart = "${pkgs.alertmanager-ntfy}/bin/alertmanager-ntfy --configs /var/lib/alertmanager-ntfy/config.yml";
373 # Run config preparation as root (+ prefix) before starting the main process
374 ExecStartPre =
375 "+"
376 + pkgs.writeShellScript "prepare-alertmanager-ntfy-config" ''
377 # Read the token from the secret file
378 TOKEN=$(cat ${config.age.secrets."ntfy-token".path})
379
380 # Generate config with the actual token
381 cat > /var/lib/alertmanager-ntfy/config.yml <<'EOF'
382 http:
383 addr: 127.0.0.1:8081
384
385 ntfy:
386 baseurl: https://ntfy.sbr.pm
387 auth:
388 token: TOKEN_PLACEHOLDER
389 notification:
390 topic: homelab
391 priority: 'status == "firing" ? "urgent" : "default"'
392 tags:
393 - tag: rotating_light
394 condition: 'status == "firing" && labels.severity == "critical"'
395 - tag: warning
396 condition: 'status == "firing" && labels.severity == "warning"'
397 - tag: "+1"
398 condition: 'status == "resolved"'
399 templates:
400 title: '{{ if eq .Status "resolved" }}✅ Resolved: {{ end }}{{ if eq .Status "firing" }}🔥 {{ end }}{{ index .Annotations "summary" }}'
401 description: '{{ index .Annotations "description" }}'
402 EOF
403 # Replace placeholder with actual token
404 ${pkgs.gnused}/bin/sed -i "s/TOKEN_PLACEHOLDER/$TOKEN/" /var/lib/alertmanager-ntfy/config.yml
405 # Make config readable by the dynamic user
406 chmod 644 /var/lib/alertmanager-ntfy/config.yml
407 '';
408 };
409 };
410
411 environment.systemPackages = with pkgs; [ yt-dlp ];
412 # mr -i u daily
413 systemd.services.mr = {
414 description = "Update configs daily";
415 requires = [ "network-online.target" ];
416 after = [ "network-online.target" ];
417
418 restartIfChanged = false;
419 unitConfig.X-StopOnRemoval = false;
420
421 serviceConfig = {
422 Type = "oneshot";
423 User = "vincent";
424 OnFailure = "status-email-root@%n.service";
425 };
426
427 path = with pkgs; [
428 git
429 mr
430 ];
431 script = ''
432 set -e
433 cd /mnt/gaia/src/configs/
434 mr -t run git reset --hard
435 mr -t u
436 '';
437
438 startAt = "daily";
439 };
440 # Open firewall for services accessible from the network
441 networking.firewall.allowedTCPPorts = [
442 8000 # Paperless-ngx web interface
443 ];
444}