feat: new diagnostics

test/package/default.nix

@@ -2,6 +2,7 @@
   (import ./migrator      { inherit system inputs self pkgs; }) //
   (import ./hemar         { inherit system inputs self pkgs; }) //
   (import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) //
+  (import ./gitea-runner-controller { inherit system inputs self pkgs; }) //
   (import ./db-tool       { inherit system inputs self pkgs; }) //
   (import ./with-attic-cache { inherit system inputs self pkgs; }) //
   (import ./element-web   { inherit system inputs self pkgs; }) //

test/package/gitea-runner-controller/default.nix

@@ -0,0 +1,54 @@
+{ inputs, self, pkgs, system, ... }:
+let
+  lib = inputs.nixpkgs.lib;
+
+  mkTestDrv = name: type:
+    if type == "directory" then
+      pkgs.runCommand "test-${name}" {} ''
+        if ! [ -f ${./test + "/${name}" + /run.sh} ]; then
+          echo "no run.sh in test/${name}"
+          exit 1
+        fi
+        mkdir -p "$out"
+        cp -r ${./test + "/${name}"}/* "$out/"
+        chmod +x "$out/run.sh"
+      ''
+    else if lib.hasSuffix ".sh" name then
+      pkgs.runCommand "test-${lib.removeSuffix ".sh" name}" {} ''
+        mkdir -p "$out"
+        install -Dm755 ${./test + "/${name}"} "$out/run.sh"
+      ''
+    else
+      null;
+
+  testDir  = builtins.readDir ./test;
+  testDrvs =
+    lib.mapAttrs' (n: v:
+      lib.nameValuePair (lib.removeSuffix ".sh" n) v
+    ) (lib.filterAttrs (_: v: v != null)
+      (lib.mapAttrs (n: t: mkTestDrv n t) testDir));
+
+  mkTest = testName: testDrv: pkgs.runCommand "gitea-runner-controller-test-${testName}"
+    {
+      nativeBuildInputs = [
+        pkgs.coreutils
+        pkgs.dash
+        pkgs.gnugrep
+        pkgs.gnused
+        pkgs.jq
+      ];
+      LOG_SH = ../../../package/gitea-runner-controller/log.sh;
+      STATE_SH = ../../../package/gitea-runner-controller/state.sh;
+      DECIDE_SH = ../../../package/gitea-runner-controller/decide.sh;
+      HCLOUD_SH = ../../../package/gitea-runner-controller/hcloud.sh;
+      GITEA_SH = ../../../package/gitea-runner-controller/gitea.sh;
+      CONTROLLER_SH = ../../../package/gitea-runner-controller/controller.sh;
+      WEBHOOK_SH = ../../../package/gitea-runner-controller/webhook.sh;
+    } ''
+      test=${testDrv}
+      ${builtins.readFile ./launch.sh}
+      mkdir -p "$out"
+    '';
+in lib.mapAttrs' (name: drv:
+  lib.nameValuePair "gitea-runner-controller-${name}" (mkTest name drv)
+) testDrvs

test/package/gitea-runner-controller/launch.sh

@@ -0,0 +1,9 @@
+#!/bin/dash
+set -eu
+
+GCR_STATE_DIR="$(mktemp -d)"
+export GCR_STATE_DIR
+export GCR_LOG=error
+trap 'rm -rf "$GCR_STATE_DIR"' EXIT INT HUP
+
+dash "$test/run.sh"

test/package/gitea-runner-controller/test/reap-diagnostics.sh

@@ -0,0 +1,72 @@
+#!/bin/dash
+set -eu
+
+. "$LOG_SH"
+. "$STATE_SH"
+. "$DECIDE_SH"
+. "$HCLOUD_SH"
+. "$GITEA_SH"
+. "$CONTROLLER_SH"
+
+gcr_state_init
+calls="$GCR_STATE_DIR/calls"
+
+gcr_gitea_job_state() {
+  case "$2" in
+    101) printf 'completed:success' ;;
+    102) printf 'completed:failure' ;;
+    *) return 1 ;;
+  esac
+}
+
+gcr_vm_public_ip() {
+  printf '192.0.2.%s' "$1"
+}
+
+gcr_vm_collect_diagnostics() {
+  printf 'diag vm=%s ip=%s job=%s reason=%s\n' "$1" "$2" "$3" "$4" >> "$calls"
+}
+
+gcr_vm_destroy() {
+  printf 'destroy vm=%s\n' "$1" >> "$calls"
+}
+
+record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}'
+record_failure='{"job_id":"102","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":42,"vm_name":"gcr-102-1","bootstrapped":true,"status":"vm_active"}'
+
+gcr_record_put 101 1 "$record_success"
+gcr_record_put 102 1 "$record_failure"
+gcr_reap_finished_jobs
+
+grep -q 'destroy vm=41' "$calls"
+grep -q 'destroy vm=42' "$calls"
+grep -q 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure' "$calls"
+if grep -q 'diag vm=41' "$calls"; then
+  printf 'success job should not collect diagnostics\n' >&2
+  exit 1
+fi
+
+test ! -e "$(gcr_record_path 101 1)"
+test ! -e "$(gcr_record_path 102 1)"
+
+calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
+calls="$calls_ip_fail"
+record_ip_fail='{"job_id":"103","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":43,"vm_name":"gcr-103-1","bootstrapped":true,"status":"vm_active"}'
+gcr_record_put 103 1 "$record_ip_fail"
+
+gcr_gitea_job_state() {
+  case "$2" in
+    103) printf 'completed:failure' ;;
+    *) return 1 ;;
+  esac
+}
+
+gcr_vm_public_ip() {
+  return 1
+}
+
+gcr_reap_finished_jobs
+
+grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail"
+grep -q 'destroy vm=43' "$calls_ip_fail"
+test ! -e "$(gcr_record_path 103 1)"

test/package/gitea-runner-controller/test/webhook-diagnostics.sh

@@ -0,0 +1,58 @@
+#!/bin/dash
+set -eu
+
+. "$LOG_SH"
+. "$STATE_SH"
+. "$DECIDE_SH"
+. "$HCLOUD_SH"
+. "$GITEA_SH"
+. "$WEBHOOK_SH"
+
+gcr_state_init
+calls="$GCR_STATE_DIR/calls"
+
+gcr_vm_public_ip() {
+  printf '192.0.2.%s' "$1"
+}
+
+gcr_vm_collect_diagnostics() {
+  printf 'diag vm=%s ip=%s job=%s reason=%s\n' "$1" "$2" "$3" "$4" >> "$calls"
+}
+
+gcr_vm_destroy() {
+  printf 'destroy vm=%s\n' "$1" >> "$calls"
+}
+
+record_success='{"job_id":"201","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":51,"vm_name":"gcr-201-1","bootstrapped":true,"status":"vm_active"}'
+record_failure='{"job_id":"202","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":52,"vm_name":"gcr-202-1","bootstrapped":true,"status":"vm_active"}'
+
+gcr_record_put 201 1 "$record_success"
+gcr_record_put 202 1 "$record_failure"
+gcr_deallocate 201 1 completed:success
+gcr_deallocate 202 1 completed:failure
+
+grep -q 'destroy vm=51' "$calls"
+grep -q 'destroy vm=52' "$calls"
+grep -q 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure' "$calls"
+if grep -q 'diag vm=51' "$calls"; then
+  printf 'success webhook should not collect diagnostics\n' >&2
+  exit 1
+fi
+
+test ! -e "$(gcr_record_path 201 1)"
+test ! -e "$(gcr_record_path 202 1)"
+
+calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
+calls="$calls_ip_fail"
+record_ip_fail='{"job_id":"203","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":53,"vm_name":"gcr-203-1","bootstrapped":true,"status":"vm_active"}'
+gcr_record_put 203 1 "$record_ip_fail"
+
+gcr_vm_public_ip() {
+  return 1
+}
+
+gcr_deallocate 203 1 completed:failure
+
+grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail"
+grep -q 'destroy vm=53' "$calls_ip_fail"
+test ! -e "$(gcr_record_path 203 1)"