Commit 8a6fa46acbc4
Changed files (4)
modules/job-notify/default.nix
@@ -0,0 +1,132 @@
+{
+ config,
+ lib,
+ pkgs,
+ ...
+}:
+
+with lib;
+let
+ cfg = config.services.job-notify;
+in
+{
+ options = {
+ services.job-notify = {
+ enable = mkEnableOption ''
+ Generic job notification service that sends ntfy notifications for systemd units
+ '';
+
+ ntfyServer = mkOption {
+ type = types.str;
+ default = "https://ntfy.sbr.pm";
+ description = ''
+ The ntfy server URL to send notifications to
+ '';
+ };
+
+ ntfyTokenFile = mkOption {
+ type = types.str;
+ description = ''
+ Path to file containing the ntfy authentication token
+ '';
+ };
+
+ defaultTopic = mkOption {
+ type = types.str;
+ default = "builds";
+ description = ''
+ Default ntfy topic for notifications that don't match any specific pattern
+ '';
+ };
+ };
+ };
+
+ config = mkIf cfg.enable {
+ systemd.services."job-notify@" = {
+ description = "Generic job notification for %i";
+ serviceConfig = {
+ Type = "oneshot";
+ ExecStart = "${pkgs.writeShellScript "job-notify" ''
+ #!/usr/bin/env bash
+ set -euo pipefail
+
+ UNIT_NAME="$1"
+ RESULT=$(${pkgs.systemd}/bin/systemctl show -p Result --value "$UNIT_NAME")
+ EXIT_CODE=$(${pkgs.systemd}/bin/systemctl show -p ExecMainStatus --value "$UNIT_NAME")
+
+ # Get execution timestamps
+ START_TIME=$(${pkgs.systemd}/bin/systemctl show -p ExecMainStartTimestamp --value "$UNIT_NAME")
+ EXIT_TIME=$(${pkgs.systemd}/bin/systemctl show -p ExecMainExitTimestamp --value "$UNIT_NAME")
+
+ # Calculate duration in seconds
+ START_EPOCH=$(${pkgs.coreutils}/bin/date -d "$START_TIME" +%s 2>/dev/null || echo "0")
+ EXIT_EPOCH=$(${pkgs.coreutils}/bin/date -d "$EXIT_TIME" +%s 2>/dev/null || echo "0")
+ DURATION=$((EXIT_EPOCH - START_EPOCH))
+
+ # Format duration as human-readable
+ if [ "$DURATION" -ge 60 ]; then
+ MINUTES=$((DURATION / 60))
+ SECONDS=$((DURATION % 60))
+ DURATION_STR="''${MINUTES}m ''${SECONDS}s"
+ else
+ DURATION_STR="''${DURATION}s"
+ fi
+
+ # Parse unit name to determine job type and topic
+ # Supports: git-<job>-<repo>-<timestamp>, build-<type>-<name>-<timestamp>, scheduled-<name>-<timestamp>
+ PREFIX=$(echo "$UNIT_NAME" | cut -d'-' -f1)
+
+ case "$PREFIX" in
+ git)
+ JOB_TYPE=$(echo "$UNIT_NAME" | cut -d'-' -f2)
+ NAME=$(echo "$UNIT_NAME" | cut -d'-' -f3)
+ TOPIC="git-builds"
+ EMOJI_SUCCESS="✅"
+ EMOJI_FAIL="❌"
+ ;;
+ build)
+ JOB_TYPE=$(echo "$UNIT_NAME" | cut -d'-' -f2)
+ NAME=$(echo "$UNIT_NAME" | cut -d'-' -f3)
+ TOPIC="remote-builds"
+ EMOJI_SUCCESS="🏗️"
+ EMOJI_FAIL="💥"
+ ;;
+ scheduled)
+ JOB_TYPE="scheduled"
+ NAME=$(echo "$UNIT_NAME" | cut -d'-' -f2)
+ TOPIC="scheduled-tasks"
+ EMOJI_SUCCESS="⏰"
+ EMOJI_FAIL="⚠️"
+ ;;
+ *)
+ JOB_TYPE="job"
+ NAME=$(echo "$UNIT_NAME" | cut -d'-' -f1)
+ TOPIC="${cfg.defaultTopic}"
+ EMOJI_SUCCESS="✅"
+ EMOJI_FAIL="❌"
+ ;;
+ esac
+
+ # Send notification
+ if [ "$RESULT" = "success" ]; then
+ ${pkgs.curl}/bin/curl -s \
+ -H "Authorization: Bearer $(${pkgs.coreutils}/bin/tr -d '\n' < ${cfg.ntfyTokenFile})" \
+ -H "Title: $EMOJI_SUCCESS $JOB_TYPE Success: $NAME ($DURATION_STR)" \
+ -H "Tags: white_check_mark,$JOB_TYPE" \
+ -H "Priority: default" \
+ -d "Job $UNIT_NAME completed successfully in $DURATION_STR (exit code: $EXIT_CODE)" \
+ "${cfg.ntfyServer}/$TOPIC" || true
+ else
+ ${pkgs.curl}/bin/curl -s \
+ -H "Authorization: Bearer $(${pkgs.coreutils}/bin/tr -d '\n' < ${cfg.ntfyTokenFile})" \
+ -H "Title: $EMOJI_FAIL $JOB_TYPE Failed: $NAME (after $DURATION_STR)" \
+ -H "Priority: high" \
+ -H "Tags: x,$JOB_TYPE,warning" \
+ -d "Job $UNIT_NAME failed after $DURATION_STR (exit code: $EXIT_CODE). Check logs: journalctl -u $UNIT_NAME" \
+ "${cfg.ntfyServer}/$TOPIC" || true
+ fi
+ ''} %i";
+ };
+ };
+ };
+}
systems/aomi/extra.nix
@@ -1,4 +1,5 @@
{
+ config,
globals,
libx,
pkgs,
@@ -28,6 +29,9 @@
# OpenShift port forwarding
./openshift-port-forward.nix
+
+ # Remote build system
+ ../../modules/job-notify
];
# Firewall is enabled in openshift-port-forward.nix
@@ -41,9 +45,25 @@
192.168.100.7 oauth-openshift.apps.ocp4.lab.home
'';
+ # Age secrets
+ age.secrets."ntfy-token" = {
+ file = ../../secrets/sakhalin/ntfy-token.age;
+ mode = "400";
+ owner = "root";
+ group = "root";
+ };
+
# TODO make it an option ? (otherwise I'll add it for all)
users.users.vincent.linger = true;
+ # Remote build system
+ services.job-notify = {
+ enable = true;
+ ntfyServer = "https://ntfy.sbr.pm";
+ ntfyTokenFile = config.age.secrets."ntfy-token".path;
+ defaultTopic = "builds";
+ };
+
services = {
logind.settings.Login = {
HandleLidSwitch = "ignore";
@@ -176,4 +196,131 @@
# Open firewall for Ollama exporter
networking.firewall.allowedTCPPorts = [ 8000 ];
+ # Builder user for remote builds
+ users.users.builder = {
+ isSystemUser = true;
+ group = "users";
+ home = "/var/lib/git-builds";
+ createHome = true;
+ openssh.authorizedKeys.keys = [
+ # This will be populated with kerkouane's SSH key for build triggering
+ ];
+ };
+
+ # Git builds directory structure
+ systemd.tmpfiles.rules = [
+ "d /var/lib/git-builds 0755 builder users -"
+ ];
+
+ # Build execution script
+ environment.etc."git-builds/execute-build.sh" = {
+ text = ''
+ #!${pkgs.bash}/bin/bash
+ set -euo pipefail
+
+ REPO_NAME="$1"
+ BUILD_TYPE="''${2:-auto}"
+ REPO_URL="vincent@kerkouane.sbr.pm:git/public/$REPO_NAME.git"
+
+ # Cache directory and worktree
+ CACHE_DIR="/var/lib/git-builds/$REPO_NAME.git"
+ BUILD_DIR="/var/lib/git-builds/$REPO_NAME-build-$(${pkgs.coreutils}/bin/date +%s)"
+
+ echo "Building $REPO_NAME (type: $BUILD_TYPE)"
+ echo "Cache: $CACHE_DIR"
+ echo "Build directory: $BUILD_DIR"
+
+ # Fetch or clone
+ if [ -d "$CACHE_DIR" ]; then
+ echo "Fetching updates for $REPO_NAME..."
+ cd "$CACHE_DIR" && ${pkgs.git}/bin/git fetch origin
+ else
+ echo "Cloning $REPO_NAME..."
+ ${pkgs.git}/bin/git clone --bare "$REPO_URL" "$CACHE_DIR"
+ fi
+
+ # Create worktree for isolated build
+ echo "Creating worktree at $BUILD_DIR..."
+ cd "$CACHE_DIR"
+ ${pkgs.git}/bin/git worktree add "$BUILD_DIR" main
+
+ cd "$BUILD_DIR"
+ echo "Working directory: $(pwd)"
+
+ # Execute build based on type
+ case "$BUILD_TYPE" in
+ nixos)
+ echo "Running NixOS build..."
+ ${pkgs.nixos-rebuild}/bin/nixos-rebuild build --flake .#aomi
+ ;;
+ make)
+ echo "Running make build..."
+ ${pkgs.gnumake}/bin/make build
+ ;;
+ docker)
+ echo "Running Docker build..."
+ ${pkgs.docker}/bin/docker build -t "$REPO_NAME:latest" .
+ ;;
+ go)
+ echo "Running Go build..."
+ ${pkgs.go}/bin/go build ./...
+ ;;
+ custom)
+ if [ -x .git-build.sh ]; then
+ echo "Running custom build script..."
+ ./.git-build.sh
+ else
+ echo "ERROR: .git-build.sh not found or not executable"
+ exit 1
+ fi
+ ;;
+ auto)
+ echo "Auto-detecting build type..."
+ if [ -f flake.nix ]; then
+ echo "Detected NixOS flake"
+ ${pkgs.nixos-rebuild}/bin/nixos-rebuild build --flake .#aomi
+ elif [ -f Makefile ]; then
+ echo "Detected Makefile"
+ ${pkgs.gnumake}/bin/make build
+ elif [ -f Dockerfile ]; then
+ echo "Detected Dockerfile"
+ ${pkgs.docker}/bin/docker build -t "$REPO_NAME:latest" .
+ elif [ -f go.mod ]; then
+ echo "Detected Go module"
+ ${pkgs.go}/bin/go build ./...
+ else
+ echo "ERROR: Could not auto-detect build type"
+ exit 1
+ fi
+ ;;
+ *)
+ echo "ERROR: Unknown build type: $BUILD_TYPE"
+ exit 1
+ ;;
+ esac
+
+ echo "Build completed successfully!"
+
+ # Cleanup worktree (keep cache)
+ echo "Cleaning up worktree..."
+ cd /
+ ${pkgs.git}/bin/git -C "$CACHE_DIR" worktree remove "$BUILD_DIR"
+ echo "Done!"
+ '';
+ mode = "0755";
+ };
+
+ # Allow builder to run systemd-run without password
+ security.sudo.extraRules = [
+ {
+ users = [ "builder" ];
+ commands = [
+ {
+ command = "${pkgs.systemd}/bin/systemd-run";
+ options = [ "NOPASSWD" ];
+ }
+ ];
+ }
+ ];
+
}
systems/kerkouane/extra.nix
@@ -222,6 +222,38 @@ in
mode = "0755";
};
+ # Remote build forwarding script
+ environment.etc."git-hooks/forward-build.sh" = {
+ text = ''
+ #!${pkgs.bash}/bin/bash
+ set -euo pipefail
+
+ REPO_NAME="$1"
+ BUILD_TYPE="''${2:-auto}"
+ TIMESTAMP=$(${pkgs.coreutils}/bin/date +%Y%m%d-%H%M%S)
+ UNIT_NAME="build-remote-''${REPO_NAME}-''${TIMESTAMP}"
+
+ echo "Forwarding build to aomi: $REPO_NAME ($BUILD_TYPE)..."
+
+ # SSH to aomi and trigger build with systemd-run
+ ${pkgs.openssh}/bin/ssh -o BatchMode=yes builder@10.100.0.17 \
+ "sudo ${pkgs.systemd}/bin/systemd-run \
+ --unit=\"$UNIT_NAME\" \
+ --description=\"Remote build: $REPO_NAME ($BUILD_TYPE)\" \
+ --property=\"OnSuccess=job-notify@\''${UNIT_NAME}.service\" \
+ --property=\"OnFailure=job-notify@\''${UNIT_NAME}.service\" \
+ --property=\"User=builder\" \
+ --property=\"Group=users\" \
+ --property=\"WorkingDirectory=/var/lib/git-builds\" \
+ /etc/git-builds/execute-build.sh \"$REPO_NAME\" \"$BUILD_TYPE\""
+
+ echo "✓ Build queued on aomi: $UNIT_NAME"
+ echo " View status: ssh aomi 'systemctl status $UNIT_NAME'"
+ echo " View logs: ssh aomi 'journalctl -u $UNIT_NAME'"
+ '';
+ mode = "0755";
+ };
+
# Example post-receive hook template
environment.etc."git-hooks/post-receive.example" = {
text = ''
@@ -231,35 +263,46 @@ in
#
# This hook uses systemd-run to execute gitmal generation in the background
# with automatic notifications via ntfy when the job completes.
+ #
+ # Optionally, it can also trigger remote builds on aomi.
set -euo pipefail
# Configuration
+ GITMAL_ENABLED="true"
GITMAL_THEME="github-dark" # Options: github-dark, github-light, dark, light, auto
+ REMOTE_BUILD_ENABLED="false" # Set to "true" to enable remote builds on aomi
+ BUILD_TYPE="nixos" # Options: nixos, make, docker, go, custom, auto
REPO_PATH="$(pwd)"
REPO_NAME=$(basename "$REPO_PATH" .git)
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
- UNIT_NAME="git-gitmal-''${REPO_NAME}-''${TIMESTAMP}"
- echo "Queuing gitmal generation for $REPO_NAME with theme: $GITMAL_THEME..."
+ # 1. Generate gitmal (local static site on kerkouane)
+ if [ "$GITMAL_ENABLED" = "true" ]; then
+ UNIT_NAME="git-gitmal-''${REPO_NAME}-''${TIMESTAMP}"
+ echo "Queuing gitmal generation for $REPO_NAME with theme: $GITMAL_THEME..."
- # Run gitmal generation in background with systemd-run
- # OnSuccess/OnFailure will trigger git-notify@.service for notifications
- sudo ${pkgs.systemd}/bin/systemd-run \
- --unit="$UNIT_NAME" \
- --description="Gitmal generation for $REPO_NAME" \
- --property="OnSuccess=git-notify@''${UNIT_NAME}.service" \
- --property="OnFailure=git-notify@''${UNIT_NAME}.service" \
- --property="User=vincent" \
- --property="Group=users" \
- --property="Environment=PATH=${pkgs.coreutils}/bin:${pkgs.git}/bin:${pkgs.gitmal}/bin" \
- --working-directory="$REPO_PATH" \
- /etc/git-hooks/generate-gitmal.sh "$REPO_PATH" "$GITMAL_THEME"
+ sudo ${pkgs.systemd}/bin/systemd-run \
+ --unit="$UNIT_NAME" \
+ --description="Gitmal generation for $REPO_NAME" \
+ --property="OnSuccess=git-notify@''${UNIT_NAME}.service" \
+ --property="OnFailure=git-notify@''${UNIT_NAME}.service" \
+ --property="User=vincent" \
+ --property="Group=users" \
+ --property="Environment=PATH=${pkgs.coreutils}/bin:${pkgs.git}/bin:${pkgs.gitmal}/bin" \
+ --working-directory="$REPO_PATH" \
+ /etc/git-hooks/generate-gitmal.sh "$REPO_PATH" "$GITMAL_THEME"
- echo "✓ Gitmal generation queued as: $UNIT_NAME"
- echo " View status: systemctl status $UNIT_NAME"
- echo " View logs: journalctl -u $UNIT_NAME"
+ echo "✓ Gitmal generation queued as: $UNIT_NAME"
+ echo " View status: systemctl status $UNIT_NAME"
+ echo " View logs: journalctl -u $UNIT_NAME"
+ fi
+
+ # 2. Trigger remote build (on aomi)
+ if [ "$REMOTE_BUILD_ENABLED" = "true" ]; then
+ /etc/git-hooks/forward-build.sh "$REPO_NAME" "$BUILD_TYPE"
+ fi
'';
mode = "0755";
};
secrets.nix
@@ -127,6 +127,7 @@ in
"secrets/sakhalin/ntfy-token.age".publicKeys = users ++ [
sakhalin
aion
+ aomi
rhea
kerkouane
];