From dc6ca7a0a926533c9450408f4da1b1eed17575e4 Mon Sep 17 00:00:00 2001 From: yukkop Date: Sun, 12 Oct 2025 03:58:43 +0000 Subject: [PATCH] =?UTF-8?q?fix(nixos):=20`senttin=C3=A8lla-probe`:=20modul?= =?UTF-8?q?e=20args?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nixos/module/hectic/service/sentinèlla.nix | 65 ++++++----- package/sentinèlla/base64.sh | 2 + package/sentinèlla/default.nix | 13 +-- package/sentinèlla/probe.sh | 4 +- .../sentinèlla/{probe-loop.sh => router.sh} | 4 +- package/sentinèlla/sentinel.sh | 110 +++++++++++++++++- 6 files changed, 152 insertions(+), 46 deletions(-) rename package/sentinèlla/{probe-loop.sh => router.sh} (97%) diff --git a/nixos/module/hectic/service/sentinèlla.nix b/nixos/module/hectic/service/sentinèlla.nix index 813c677..14b15da 100644 --- a/nixos/module/hectic/service/sentinèlla.nix +++ b/nixos/module/hectic/service/sentinèlla.nix @@ -19,13 +19,15 @@ in { probe = { enable = lib.mkEnableOption "enable sentinèlla probe services, that provides endpoints for server status check"; urls = lib.mkOption { - type = lib.types.port; + type = with lib.types; listOf str; + default = []; description = '' urls to check ''; }; authFile = lib.mkOption { - type = lib.types.path; + type = with lib.types; nullOr path; + default = null; example = '' config.sops.secrets."name-of-service/sentinèlla-probe".path ''; @@ -34,19 +36,22 @@ in { ''; }; volumes = lib.mkOption { - type = lib.types.port; + type = with lib.types; listOf str; + default = []; description = '' volumes to check ''; }; port = lib.mkOption { type = lib.types.port; + default = 5988; description = '' service's port ''; }; environmentPath = lib.mkOption { - type = lib.types.path; + type = with lib.types; nullOr path; + default = null; example = '' config.sops.secrets."name-of-service/environment".path ''; @@ -77,34 +82,36 @@ in { }; config = lib.mkMerge [ (lib.mkIf cfg.probe.enable { - systemd.services."sentinèlla-probe" = { + systemd.services."sentinella-probe" = { description = "Hectic server health check"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "simple"; - ExecStart = "${self.packages.${system}."sentinèlla"}/bin/probe"; - EnvironmentFile = cfg.probe.environmentPath; - Environment = (if cfg.probe.urls != null then [ - "URLS=${cfg.probe.urls}" - ] else []) ++ (if cfg.probe.volumes != null then [ - "VOLUMES=${cfg.volumes}" - ] else []) ++ (if cfg.probe.port != null then [ - "PORT=${builtins.toString cfg.probe.port}" - ] else []); - Restart = "always"; - RestartSec = "5s"; - - # Shutdown configuration - TimeoutStopSec = "30s"; - KillSignal = "SIGTERM"; - KillMode = "mixed"; - - # Security and process management - RemainAfterExit = false; - StandardOutput = "journal"; - StandardError = "journal"; - }; + serviceConfig = lib.mkMerge [ + { + Type = "simple"; + ExecStart = "${self.packages.${system}."sentinèlla"}/bin/probe"; + Environment = [ + "URLS=${lib.concatStringsSep "," cfg.probe.urls}" + "VOLUMES=${lib.concatStringsSep "," cfg.probe.volumes}" + "PORT=${builtins.toString cfg.probe.port}" + ]; + Restart = "always"; + RestartSec = "5s"; + + # Shutdown configuration + TimeoutStopSec = "30s"; + KillSignal = "SIGTERM"; + KillMode = "mixed"; + + # Security and process management + RemainAfterExit = false; + StandardOutput = "journal"; + StandardError = "journal"; + } + (if cfg.probe.environmentPath != null then { + EnvironmentFile = cfg.probe.environmentPath; + } else {}) + ]; }; }) (lib.mkIf cfg.sentinel.enable { diff --git a/package/sentinèlla/base64.sh b/package/sentinèlla/base64.sh index e133626..b2ae1ce 100644 --- a/package/sentinèlla/base64.sh +++ b/package/sentinèlla/base64.sh @@ -1,3 +1,5 @@ +#!/bin/dash + mod="${1:?}" case "$mod" in diff --git a/package/sentinèlla/default.nix b/package/sentinèlla/default.nix index 90c1fbf..fa6d8fb 100644 --- a/package/sentinèlla/default.nix +++ b/package/sentinèlla/default.nix @@ -1,4 +1,4 @@ -{ symlinkJoin, writeShellApplication, socat, dash, hectic, curl }: +{ symlinkJoin, writeShellApplication, socat, dash, hectic, curl, gawk }: let shell = "${dash}/bin/dash"; bashOptions = [ @@ -13,19 +13,18 @@ let text = builtins.readFile ./base64.sh; }; - # TODO: writeDashApplication probe = hectic.writeShellApplication { inherit shell bashOptions; name = "probe"; - runtimeInputs = [ socat dash probe-loop ]; + runtimeInputs = [ socat dash router ]; text = builtins.readFile ./probe.sh; }; - probe-loop = hectic.writeShellApplication { + router = hectic.writeShellApplication { inherit shell bashOptions; - name = "probe-loop"; - runtimeInputs = [ base64 ]; - text = builtins.readFile ./probe-loop.sh; + name = "router"; + runtimeInputs = [ base64 gawk ]; + text = builtins.readFile ./router.sh; }; sentinel = hectic.writeShellApplication { diff --git a/package/sentinèlla/probe.sh b/package/sentinèlla/probe.sh index a89d185..32d0294 100644 --- a/package/sentinèlla/probe.sh +++ b/package/sentinèlla/probe.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env dash +#!/bin/dash socat -V >/dev/null dash -c 'echo ok' >/dev/null -socat -T5 -t5 TCP-LISTEN:"${PORT:-5988}",reuseaddr,fork EXEC:"probe-loop" +socat -T5 -t5 TCP-LISTEN:"${PORT:-5988}",reuseaddr,fork EXEC:"router" diff --git a/package/sentinèlla/probe-loop.sh b/package/sentinèlla/router.sh similarity index 97% rename from package/sentinèlla/probe-loop.sh rename to package/sentinèlla/router.sh index 66143e8..7460434 100644 --- a/package/sentinèlla/probe-loop.sh +++ b/package/sentinèlla/router.sh @@ -1,7 +1,7 @@ -#!/usr/bin/env dash +#!/bin/dash # router.sh — POSIX sh HTTP backend (for socat) -# usage: socat -T5 -t5 TCP-LISTEN:${port},reuseaddr,fork EXEC:"sh ${currentfile}" +# usage: socat -T5 -t5 TCP-LISTEN:${port},reuseaddr,fork EXEC:"dash ${currentfile}" # Routes: # GET /status -> check $URLS (0/0 if unset) # GET /disk -> check $VOLUMES (all if unset) diff --git a/package/sentinèlla/sentinel.sh b/package/sentinèlla/sentinel.sh index 89ea249..56f5d78 100644 --- a/package/sentinèlla/sentinel.sh +++ b/package/sentinèlla/sentinel.sh @@ -1,11 +1,109 @@ #!/bin/dash +# sentinel.sh — polls probe backends (/status) and notifies on status change via Telegram +# Env: +# SERVERS="http://host1:8080,http://host2:8080" +# TOKENS="-,b64token2" # CSV aligned with SERVERS; "-" means no auth +# TOKEN="..." # Telegram bot token +# CHAT_ID="..." # Telegram chat id +# TIMEOUT=5 # curl timeout seconds (default 5) +# POLLING_INTERVAL_SEC=3 # default 3 +# STATE_DIR=/tmp/sentinel # default /tmp/sentinel -TOKEN=8448534574:AAEvsdQqhUDu3RVRJWDGIVeqRmXlB0Dqn1Q -CHAT_ID=380055934 +set -eu + +TIMEOUT=${TIMEOUT:-5} POLLING_INTERVAL_SEC=${POLLING_INTERVAL_SEC:-3} +STATE_DIR=${STATE_DIR:-/tmp/sentinel} +SERVERS=${SERVERS:-} +TOKENS=${TOKENS:-} +TOKEN=${TOKEN:-} +CHAT_ID=${CHAT_ID:-} -while true; do - curl -s -X POST "https://api.telegram.org/bot${TOKEN}/sendMessage" \ - -d "chat_id=${CHAT_ID}" \ - -d text="your message" +[ -n "$SERVERS" ] || { printf >&2 'SERVERS not set\n'; exit 1; } + +# If TOKENS unset, synthesize "-" for each server +if [ -z "$TOKENS" ]; then + n=$(printf '%s\n' "$SERVERS" | tr -cd ',' | wc -c | awk '{print $1+1}') + TOKENS=$(awk -v n="$n" 'BEGIN{for(i=1;i<=n;i++){printf("-"); if(i echo idx-th field (1-based) from CSV string VAR +get_csv() { + # shellcheck disable=SC2001 + printf '%s' "$1" | sed 's/,/\n/g' | awk -v n="$2" 'NR==n{print; exit}' +} + +notify() { + msg=$1 + if [ -n "$TOKEN" ] && [ -n "$CHAT_ID" ]; then + curl -sS -m "$TIMEOUT" -X POST "https://api.telegram.org/bot${TOKEN}/sendMessage" \ + -d "chat_id=${CHAT_ID}" \ + --data-urlencode "text=${msg}" >/dev/null || printf >&2 'notify failed: %s\n' "$msg" + else + printf >&2 '%s\n' "$msg" + fi +} + +sid() { printf '%s' "$1" | cksum | awk '{print $1}'; } + +parse_summary() { + sed -n 's/.*"summary":{"total":\([0-9][0-9]*\),"ok":\([0-9][0-9]*\)}.*/\1 \2/p' +} + +list_failures() { + awk ' + BEGIN{FS="\""; u=""; c=""} + /"url":/ {u=$4} + /"code":/ {c=$0; sub(/.*"code":/,"",c); sub(/,.*/,"",c)} + /"ok":false/ { if(u!=""){ printf "%s(%s) ", u, c; u=""; c="" } } + ' +} + +# --- main loop --- +while :; do + i=1 + while :; do + srv=$(get_csv "$SERVERS" "$i") || true + [ -n "${srv:-}" ] || break + tok=$(get_csv "$TOKENS" "$i") || tok="-" + + url="${srv%/}/status" + auth_h="" + [ "${tok}" != "-" ] && [ -n "${tok}" ] && auth_h="-H Authorization: Basic\ $tok" + + tmpb=$(mktemp) || exit 1 + code=$(sh -c "curl -sS -m \"$TIMEOUT\" -w '%{http_code}' -o \"$tmpb\" $auth_h \"$url\"") || code="000" + body=$(cat "$tmpb"); rm -f "$tmpb" + + ok="down"; tot=0; good=0 + if [ "$code" = "200" ]; then + s=$(printf '%s' "$body" | parse_summary || true) + [ -n "$s" ] && { tot=${s%% *}; good=${s#* }; } + [ "$tot" -eq "$good" ] && ok="up" + fi + + msg_prefix=$( [ "$ok" = "up" ] && printf 'OK' || printf 'FAIL' ) + fail_list="" + if [ "$ok" = "down" ] && [ -n "$body" ]; then + fails=$(printf '%s' "$body" | list_failures | sed 's/[ ]$//') + [ -n "$fails" ] && fail_list=" — ${fails}" + fi + msg=$(printf '%s: %s [%s/%s]%s' "$msg_prefix" "$srv" "$good" "$tot" "$fail_list") + + sfile="${STATE_DIR}/$(sid "$srv").state" + last=""; [ -f "$sfile" ] && last=$(cat "$sfile") + cur="${ok}:${good}/${tot}:${code}" + if [ "$cur" != "$last" ]; then + notify "$msg" + printf '%s' "$cur" >"$sfile" + fi + + i=$((i+1)) + done + + sleep "$POLLING_INTERVAL_SEC" done