feat: zero-idle gite runners infra

infra/gitea-runners/k8s/statefulset.yaml

@@ -8,7 +8,8 @@ metadata:
     app.kubernetes.io/part-of: gitea-actions
 spec:
   serviceName: gitea-runner
-  replicas: 1
+  # Rollback-only pool. Keep manifest present, default replicas 0.
+  replicas: 0
   podManagementPolicy: Parallel
   selector:
     matchLabels:

infra/gitea-runners/runbook.md

@@ -501,3 +501,117 @@ Do not release unless the following evidence files exist and are readable:
 
 If any evidence file is missing, stop and collect it before treating the runbook
 as complete.
+
+## Ephemeral VM runner cutover
+
+This section governs replacing the fixed K8s runner pool with the
+ephemeral-VM controller (`package/gitea-runner-controller`) on this host.
+The K8s pool above remains rollback-only until cutover is explicitly accepted.
+
+### Operator gates (all three required before enable)
+
+1. Secrets — add to `sus/gitea-runners.yaml` under
+   `gitea/hectic-lab/controller/*`:
+
+   ```sh
+   sops sus/gitea-runners.yaml
+   # add keys:
+   #   gitea:
+   #     hectic-lab:
+   #       controller:
+   #         hcloud-token:    <Hetzner API token, VM create/destroy scope>
+   #         webhook-secret:  <random 32+ bytes; also set as Gitea webhook secret>
+   #         admin-token:     <Gitea token with admin:runner scope for stale cleanup>
+   ```
+
+   The org registration token key
+   `gitea/hectic-lab/org-runner-registration-token` already exists and is reused.
+
+2. Base image — build the MicroOS snapshot and record its id:
+
+   ```sh
+   nix develop .#gitea-runners -c gitea-runners-build-microos-snapshots x86
+   hcloud image list --selector '' -o json | jq '.[] | select(.type=="snapshot")'
+   ```
+
+3. DNS — A record `runners.hectic-lab.com -> 128.140.75.58` (ACME needs it).
+
+### Enable
+
+```sh
+# nixos/system/hectic-lab/hectic-lab.nix: resolve the FIXME block
+hectic.services.gitea-runner-controller = {
+  enable  = true;
+  imageId = "<snapshot-id-from-gate-2>";
+};
+nixos-rebuild --target root@128.140.75.58 switch
+systemctl status gitea-runner-controller.service gitea-runner-webhook.service
+```
+
+### Register the Gitea webhook
+
+Org-level (preferred) or per-repo, on `https://gitea.hectic-lab.com`:
+
+- URL: `https://runners.hectic-lab.com/`
+- Method: `POST`, content type: JSON
+- Secret: value of `gitea/hectic-lab/controller/webhook-secret`
+- Trigger events: `Workflow jobs` only (`workflow_job`)
+
+### Pre-flight verification (before first real job)
+
+```sh
+curl -fsS https://runners.hectic-lab.com/ -o /dev/null -w '%{http_code}\n' # any 4xx from handler = reachable
+journalctl -u gitea-runner-webhook -n 20 --no-pager
+hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed")] | length' # expect 0
+```
+
+Zero managed VMs at idle is the steady-state assertion.
+
+### End-to-end acceptance (Task 9)
+
+Trigger `.gitea/workflows/runner-nix-smoke.yaml` via workflow_dispatch, then:
+
+```sh
+watch_labels() { hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed") | {id,name,labels}]'; }
+watch_labels                                   # exactly one VM while queued/running
+journalctl -f -u gitea-runner-controller       # vm-created / vm-destroyed events
+watch_labels                                   # expect [] after completion
+curl -fsS -H "Authorization: token $ADMIN" \
+  https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners \
+  | jq '[.entries[] | select(.name | startswith("gcr-"))] | length'  # expect 0
+```
+
+Failure paths to verify identically: duplicate delivery (send same webhook twice
+via Gitea UI "Test delivery"), cancelled job, unknown-label job.
+
+### Rollback
+
+K8s rollback pool now defaults to deleted state:
+
+- `infra/gitea-runners/k8s/statefulset.yaml` keeps `replicas: 0`
+- old kube-hetzner nodes may be deleted to preserve zero idle cost
+- PVCs and IaC remain for manual rollback only
+
+Re-enable sequence:
+
+```sh
+# 1. stop ephemeral path
+sed -i 's/hectic.services.gitea-runner-controller = {.*}/\/* disabled *\//' \
+  nixos/system/hectic-lab/hectic-lab.nix   # or set enable = false
+nixos-rebuild --target root@128.140.75.58 switch
+
+# 2. reprovision old kube-hetzner nodes when they were deleted:
+tofu -chdir=infra/gitea-runners/opentofu apply
+
+# 3. restore kubeconfig / cluster access, then re-enable K8s runner pool:
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+```
+
+Any surviving ephemeral VMs after step 1 must be destroyed manually once:
+
+```sh
+hcloud server list -o json \
+  | jq -r '.[] | select(.labels["gitea-runner-controller"]=="managed") | .id' \
+  | xargs -r -n1 hcloud server delete
+```

nixos/module/hectic/service/gitea-runner-controller.nix

@@ -0,0 +1,230 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  system = pkgs.stdenv.hostPlatform.system;
+  cfg = config.hectic.services.gitea-runner-controller;
+
+  secretPrefix = "gitea-runner-controller";
+  mkControllerSecret = name: {
+    name = "${secretPrefix}/${name}";
+    value = {
+      sopsFile = flake + "/sus/gitea-runners.yaml";
+      key      = "gitea/hectic-lab/controller/${name}";
+    };
+  };
+
+  controllerSecrets = builtins.listToAttrs (map mkControllerSecret [
+    "hcloud-token"
+    "webhook-secret"
+    "admin-token"
+    "ssh-private-key"
+  ]);
+
+  # Registration token is shared with the existing K8s/local runner setup.
+  registrationTokenPath = config.sops.secrets."gitea-runner/org-registration-token".path;
+
+  commonEnvironment =
+    [
+      "GCR_STATE_DIR=/var/lib/gitea-runner-controller"
+      "GCR_GITEA_URL=${cfg.giteaBaseUrl}"
+      "GCR_ALLOWED_REPOS=${lib.concatStringsSep "," cfg.allowedRepos}"
+      "GCR_CONCURRENCY_CAP=${toString cfg.concurrencyCap}"
+      "GCR_PER_REPO_CAP=${toString cfg.perRepoCap}"
+      "GCR_RECONCILE_INTERVAL_SEC=${toString cfg.reconcileIntervalSec}"
+      "GCR_BUDGET_EUR_MONTHLY=${cfg.budgetEurMonthly}"
+      "GCR_HETZNER_LOCATION=${cfg.hetznerLocation}"
+      "HCLOUD_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/hcloud-token".path}"
+      "GITEA_WEBHOOK_SECRET_FILE=${config.sops.secrets."${secretPrefix}/webhook-secret".path}"
+      "GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}"
+      "GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}"
+      "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}"
+      "GCR_NIX_VERSION=${cfg.nixVersion}"
+      "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}"
+      "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}"
+      "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}"
+    ]
+    ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ];
+in
+{
+  options = {
+    hectic.services.gitea-runner-controller = {
+      enable = lib.mkEnableOption "gitea-runner-controller — ephemeral Hetzner VM runner controller";
+      listenAddr = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+        description = "Address the webhook listener binds to.";
+      };
+      listenPort = lib.mkOption {
+        type = lib.types.port;
+        default = 8787;
+        description = "Port the webhook listener binds to.";
+      };
+      webhookHost = lib.mkOption {
+        type = lib.types.str;
+        default = "runners.hectic-lab.com";
+        description = "Public vhost Gitea delivers webhooks to. Requires a DNS A record to this host.";
+      };
+      giteaBaseUrl = lib.mkOption {
+        type = lib.types.str;
+        default = "https://gitea.hectic-lab.com";
+        description = "Public Gitea URL runners register against.";
+      };
+      allowedRepos = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ "hectic-lab/util.nix" ];
+        description = "Repos whose workflow_job events may trigger VM creation.";
+      };
+      concurrencyCap = lib.mkOption {
+        type = lib.types.int;
+        default = 2;
+        description = "Maximum simultaneously running ephemeral VMs (global).";
+      };
+      perRepoCap = lib.mkOption {
+        type = lib.types.int;
+        default = 1;
+        description = "Maximum concurrent ephemeral VMs per repo.";
+      };
+      reconcileIntervalSec = lib.mkOption {
+        type = lib.types.int;
+        default = 60;
+        description = "Seconds between reconciliation ticks.";
+      };
+      budgetEurMonthly = lib.mkOption {
+        type = lib.types.str;
+        default = "15";
+        description = "Soft monthly EUR ceiling for estimated VM spend.";
+      };
+      hetznerLocation = lib.mkOption {
+        type = lib.types.str;
+        default = "nbg1";
+        description = "Hetzner location for ephemeral VMs.";
+      };
+      imageId = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        example = "174108912";
+        description = ''
+          Hetzner image/snapshot id for ephemeral VMs (MicroOS base from
+          gitea-runners-build-microos-snapshots). Controller refuses VM
+          creation while null.
+        '';
+      };
+      actRunnerVersion = lib.mkOption {
+        type = lib.types.str;
+        default = "v0.2.11";
+        description = "act_runner release tag downloaded at VM bootstrap.";
+      };
+      actRunnerSha256 = lib.mkOption {
+        type = lib.types.str;
+        default = "7a5e833793286bbfd9b59ce682bd41fc3f1c096bae1bb2a09b66ab2f6dacf90c";
+        description = "sha256 of the pinned act_runner linux-amd64 binary, verified at bootstrap.";
+      };
+      nixVersion = lib.mkOption {
+        type = lib.types.str;
+        default = "2.28.3";
+        description = "Nix release installed from the official static tarball at bootstrap.";
+      };
+      nixTarballSha256 = lib.mkOption {
+        type = lib.types.str;
+        default = "85d1847d06d5d56167796d3f61cd992908de84584db3e700da031a782b59ea22";
+        description = "sha256 of the pinned Nix x86_64-linux tarball, verified at bootstrap.";
+      };
+      debugSshPublicKey = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        description = ''
+          Injected into every ephemeral VM (project ssh-key "yukkop@nixos"
+          carries the matching public key; this value is informational and
+          used by gcr_bootstrap_script documentation).
+        '';
+      };
+      bootstrapSshPrivateKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          Path (sops-rendered) to the SSH private key the controller uses to
+          push bootstrap into ephemeral VMs. Public half must be registered
+          as Hetzner project ssh-key "yukkop@nixos".
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.imageId != null;
+        message = "gitea-runner-controller: imageId must be set to a MicroOS snapshot id before enabling";
+      }
+    ];
+
+    sops.secrets = controllerSecrets;
+
+    systemd.services.gitea-runner-controller = {
+      description = "Gitea ephemeral runner controller — reconcile loop";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${self.packages.${system}.gitea-runner-controller}/bin/gitea-runner-controller";
+        Restart = "always";
+        RestartSec = "5s";
+        StateDirectory = "gitea-runner-controller";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        StandardOutput = "journal";
+        StandardError = "journal";
+        Environment = commonEnvironment;
+      };
+    };
+
+    systemd.services.gitea-runner-webhook = {
+      description = "Gitea ephemeral runner webhook receiver";
+      after = [
+        "network.target"
+        "gitea-runner-controller.service"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      partOf = [ "gitea-runner-controller.service" ];
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${self.packages.${system}.gitea-runner-controller}/bin/gitea-runner-webhook";
+        Restart = "always";
+        RestartSec = "2s";
+        StateDirectory = "gitea-runner-controller";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        StandardOutput = "journal";
+        StandardError = "journal";
+        Environment = commonEnvironment ++ [
+          "GCR_LISTEN_ADDR=${cfg.listenAddr}"
+          "GCR_LISTEN_PORT=${toString cfg.listenPort}"
+        ];
+      };
+    };
+
+    services.nginx.virtualHosts."${cfg.webhookHost}" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        extraConfig = ''
+          proxy_pass http://${cfg.listenAddr}:${toString cfg.listenPort};
+          proxy_read_timeout 30s;
+        '';
+      };
+    };
+  };
+}

nixos/system/hectic-lab/hectic-lab.nix

@@ -92,6 +92,19 @@ in {
     services.matrix = {
       enable = false;
     };
+    services.gitea-runner-controller = {
+      # NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job).
+      # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover".
+      enable  = true;
+      imageId = "424558114"; # MicroOS x86 + Hetzner datasource dhcpcd fix
+      allowedRepos = [
+        "hectic-lab/util.nix"
+        "hectic-lab/runner-clean"
+        "hectic-lab/runner-clean2"
+      ];
+      # FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable.
+      debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ yukkop@nixos";
+    };
   };
 
   # NOTE(yukkop): disk was provisioned by Hetzner rescue image, disko was never
@@ -294,6 +307,8 @@ in {
     virtualHosts."gitea.${domain}" = {
       enableACME = true;
       forceSSL = true;
+      # NOTE(yukkop): allow large git pushes over HTTPS.
+      extraConfig = "client_max_body_size 512m;";
       locations."/" = {
         extraConfig = ''
           proxy_pass     http://127.0.0.1:11011/;
@@ -323,7 +338,7 @@ in {
       };
     };
     gitea-actions-runner.instances.${giteaRunnerInstance} = {
-      enable    = true;
+      enable    = false;
       name      = giteaRunnerInstance;
       url       = "https://gitea.${domain}";
       tokenFile = giteaRunnerTokenEnv;

package/default.nix

@@ -144,6 +144,7 @@ in {
   support-bot                  = pkgs.callPackage ./support-bot                       {};
   gitea-heatmap                = pkgs.callPackage ./gitea                             {};
   gitea-runner-nix-image       = pkgs.callPackage ./gitea-runner-nix-image            {};
+  gitea-runner-controller      = pkgs.callPackage ./gitea-runner-controller           {};
   nix-derivation-hash          = pkgs.callPackage ./nix-derivation-hash               {};
   "sentinèlla"                 = pkgs.callPackage (./. + "/sentinèlla")               {};
   deploy                       = pkgs.callPackage ./deploy                            { inherit inputs; };

package/gitea-runner-controller/controller.sh

@@ -0,0 +1,219 @@
+#!/bin/dash
+# Reconcile loop for gitea-runner-controller.
+# Owns: TTL sweep, orphan-VM sweep, deferred-job retry, stale-runner dereg,
+# startup convergence. Runs forever under systemd; webhook service is separate.
+
+gcr_ttl_grace_sec() {
+    printf '%s' "$((10 * 60))"
+}
+
+gcr_record_age_sec() {
+    created_at="$(gcr_record_field "$1" created_at)"
+    now="$(date -u '+%s')"
+    case "$created_at" in
+        ''|*[!0-9]*) printf '%s' 999999 ;;
+        *) printf '%s' "$((now - created_at))" ;;
+    esac
+}
+
+gcr_sweep_ttl() {
+    for f in $(gcr_active_records); do
+        rec="$(cat "$f")"
+        status="$(gcr_record_field "$rec" status)"
+        [ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue
+
+        job_id="$(gcr_record_field "$rec" job_id)"
+        attempt="$(gcr_record_field "$rec" run_attempt)"
+        ttl_min="$(gcr_record_field "$rec" ttl_min)"
+        case "$ttl_min" in ''|*[!0-9]*) continue ;; esac
+
+        max_sec="$((ttl_min * 60 + $(gcr_ttl_grace_sec)))"
+        age="$(gcr_record_age_sec "$rec")"
+        if [ "$age" -gt "$max_sec" ]; then
+            vm_id="$(gcr_record_field "$rec" vm_id)"
+            gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s"
+            if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
+                gcr_vm_destroy "$vm_id" || true
+                gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
+            fi
+            gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}"
+            gcr_record_del "$job_id" "$attempt"
+            gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
+        fi
+    done
+}
+
+gcr_sweep_orphan_vms() {
+    vms_json="$(gcr_vm_list_managed)" || return 0
+    count="$(printf '%s' "$vms_json" | jq 'length')"
+    i=0
+    while [ "$i" -lt "$count" ]; do
+        vm="$(printf '%s' "$vms_json" | jq -c ".[$i]")"
+        vm_id="$(printf '%s' "$vm" | jq -r '.id')"
+        jid="$(printf '%s' "$vm" | jq -r '.labels["gcr.job-id"] // ""')"
+        att="$(printf '%s' "$vm" | jq -r '.labels["gcr.run-attempt"] // ""')"
+
+        known=""
+        if [ -n "$jid" ] && [ -n "$att" ]; then
+            rec="$(gcr_record_get "$jid" "$att")"
+            [ -n "$rec" ] && known=1
+        fi
+
+        if [ -z "$known" ]; then
+            gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy"
+            gcr_vm_destroy "$vm_id" || true
+            gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}"
+        fi
+        i=$((i + 1))
+    done
+}
+
+gcr_alloc_deferred() {
+    job_id="$1"; attempt="$2"
+
+    rec="$(gcr_record_get "$job_id" "$attempt")"
+    [ -n "$rec" ] || return 0
+    [ "$(gcr_record_field "$rec" status)" = "deferred" ] || return 0
+
+    repo="$(gcr_record_field "$rec" repo)"
+    label="$(gcr_record_field "$rec" label)"
+
+    profile="$(gcr_label_profile "$label")" || return 0
+    set -- $profile
+    server_type="$1"; ttl_min="$2"; rate="$3"
+
+    active="$(gcr_count_active)"
+    repo_active="$(gcr_count_active_repo "$repo")"
+    [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] && return 0
+    [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ] && return 0
+
+    gcr_budget_add "$rate" "$ttl_min" || return 0
+    reg_token="$(gcr_gitea_registration_token)" || return 0
+
+    key="$(gcr_alloc_key "$job_id" "$attempt")"
+    gcr_lock_acquire "$key" || return 0
+
+    vm_name="gcr-${job_id}-${attempt}"
+    vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
+        "$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || {
+        gcr_lock_release "$key"
+        return 0
+    }
+
+    rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
+        --arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \
+        --arg vn "$vm_name" --arg ttl "$ttl_min" \
+        '{job_id:$j, run_attempt:$a, repo:$r, label:$l,
+          created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
+          vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
+    gcr_record_put "$job_id" "$attempt" "$rec"
+    gcr_lock_release "$key"
+    gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min,\"via\":\"deferred-retry\"}"
+    gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id"
+}
+
+gcr_retry_deferred() {
+    for f in $(gcr_active_records); do
+        rec="$(cat "$f")"
+        [ "$(gcr_record_field "$rec" status)" = "deferred" ] || continue
+        gcr_alloc_deferred \
+            "$(gcr_record_field "$rec" job_id)" \
+            "$(gcr_record_field "$rec" run_attempt)"
+    done
+}
+
+gcr_sweep_stale_runners() {
+    runners="$(gcr_gitea_list_runners)" || return 0
+    # Here-doc instead of pipe: dash runs pipe tails in a subshell, which
+    # would strand gcr_event/audit writes from the caller's perspective.
+    while read -r rid rname; do
+        [ -n "${rid:-}" ] || continue
+        case "$rname" in
+            gcr-*) ;;
+            *) continue ;;
+        esac
+
+        # gcr-<job>-<attempt>: alive iff a matching active/pending record exists.
+        rest="${rname#gcr-}"
+        jid="${rest%-*}"
+        att="${rest##*-}"
+        rec=""
+        case "$jid" in *[!0-9]*|"") rec="" ;;
+            *) case "$att" in *[!0-9]*|"") rec="" ;;
+                   *) rec="$(gcr_record_get "$jid" "$att")" ;;
+               esac ;;
+        esac
+
+        if [ -z "$rec" ]; then
+            gcr_log warn --ns=sweep "stale runner registration id=$rid name=$rname -> delete"
+            if gcr_gitea_delete_runner "$rid"; then
+                gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"runner_id\":$rid,\"name\":\"$rname\"}"
+            else
+                gcr_log error --ns=sweep "failed deleting runner id=$rid"
+            fi
+        fi
+    done <<EOF
+$runners
+EOF
+}
+
+gcr_vm_public_ip() {
+    # gcr_vm_public_ip SERVER_ID -> ipv4 or empty
+    if gcr_hcloud_req GET "/servers/$1"; then
+        jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY"
+    fi
+}
+
+# Runs SSH-push bootstrap for VMs that were created but not yet provisioned.
+# Registration token is fetched fresh per attempt (short-lived usefulness).
+gcr_bootstrap_pending() {
+    for f in $(gcr_active_records); do
+        rec="$(cat "$f")"
+        [ "$(gcr_record_field "$rec" status)" = "pending_vm" ] || continue
+        [ "$(gcr_record_field "$rec" bootstrapped)" = "true" ] && continue
+
+        job_id="$(gcr_record_field "$rec" job_id)"
+        attempt="$(gcr_record_field "$rec" run_attempt)"
+        label="$(gcr_record_field "$rec" label)"
+        vm_id="$(gcr_record_field "$rec" vm_id)"
+        runner_name="$(gcr_record_field "$rec" vm_name)"
+
+        ip="$(gcr_vm_public_ip "$vm_id")"
+        [ -n "$ip" ] || continue
+
+        reg_token="$(gcr_gitea_registration_token)" || continue
+        ttl_min="$(gcr_record_field "$rec" ttl_min)"
+
+        gcr_log info --ns=alloc "bootstrapping vm=$vm_id ip=$ip job=$job_id"
+        if gcr_vm_bootstrap_ssh "$ip" "$label" "$reg_token" "$runner_name"; then
+            rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | .ip = $ip' --arg ip "$ip")"
+            gcr_record_put "$job_id" "$attempt" "$rec"
+            gcr_event "vm-bootstrapped" "$job_id" "{\"vm_id\":$vm_id,\"ip\":\"$ip\"}"
+        else
+            gcr_log warn --ns=alloc "bootstrap failed vm=$vm_id (retry next tick)"
+        fi
+    done
+}
+
+gcr_tick() {
+    gcr_sweep_ttl
+    gcr_sweep_orphan_vms
+    gcr_retry_deferred
+    gcr_bootstrap_pending
+    gcr_sweep_stale_runners
+}
+
+gcr_main() {
+    : "${GCR_RECONCILE_INTERVAL_SEC:=60}"
+    gcr_state_init
+
+    gcr_log info --ns=core "controller starting, interval=${GCR_RECONCILE_INTERVAL_SEC}s state=$GCR_STATE_DIR"
+    gcr_tick
+
+    while :; do
+        sleep "$GCR_RECONCILE_INTERVAL_SEC"
+        if ! gcr_tick; then
+            gcr_log error --ns=core "tick failed, retrying next interval"
+        fi
+    done
+}

package/gitea-runner-controller/decide.sh

@@ -0,0 +1,70 @@
+#!/bin/dash
+# Allocation decision for gitea-runner-controller.
+# Fail-closed: anything not explicitly allowed here is refused.
+#
+# gcr_decide LABEL REPO -> prints "<server_type> <ttl_min> <rate_eur_h>" and
+# returns 0 when allowed; returns 1 with reason on stderr otherwise.
+
+gcr_label_profile() {
+    case "$1" in
+        nix)           printf 'cx33 180 0.008' ;;
+        ubuntu-latest) printf 'cx33 60 0.008'  ;;
+        *) return 1 ;;
+    esac
+}
+
+gcr_repo_allowed() {
+    repo="$1"
+    oldIFS="$IFS"
+    IFS=,
+    for allowed in ${GCR_ALLOWED_REPOS:-}; do
+        if [ "$allowed" = "$repo" ]; then
+            IFS="$oldIFS"
+            return 0
+        fi
+    done
+    IFS="$oldIFS"
+    return 1
+}
+
+gcr_decide() {
+    label="$1"; repo="$2"
+
+    if ! gcr_repo_allowed "$repo"; then
+        gcr_log warn --ns=decide "repo not allowed: $repo" 
+        return 1
+    fi
+
+    # Multi-label jobs are out of MVP scope: ambiguous VM profile mapping.
+    if ! profile="$(gcr_label_profile "$label")"; then
+        gcr_log warn --ns=decide "unknown or unsupported label: $label"
+        return 1
+    fi
+
+    printf '%s\n' "$profile"
+}
+
+gcr_count_active() {
+    count=0
+    for f in $(gcr_active_records); do
+        status="$(gcr_record_field "$(cat "$f")" status)"
+        case "$status" in
+            pending_vm|vm_active) count=$((count + 1)) ;;
+        esac
+    done
+    printf '%s' "$count"
+}
+
+gcr_count_active_repo() {
+    repo="$1"
+    count=0
+    for f in $(gcr_active_records); do
+        rec="$(cat "$f")"
+        case "$(gcr_record_field "$rec" status)" in
+            pending_vm|vm_active) ;;
+            *) continue ;;
+        esac
+        [ "$(gcr_record_field "$rec" repo)" = "$repo" ] && count=$((count + 1))
+    done
+    printf '%s' "$count"
+}

package/gitea-runner-controller/default.nix

@@ -0,0 +1,92 @@
+{
+  symlinkJoin,
+  hectic,
+  dash,
+  socat,
+  curl,
+  jq,
+  coreutils,
+  gawk,
+  gnugrep,
+  gnused,
+  openssl,
+}:
+let
+  shell = "${dash}/bin/dash";
+  bashOptions = [
+    "errexit"
+    "nounset"
+  ];
+  # SC2329: lib units are sourced wholesale into both binaries; the reconciler
+  # and webhook each leave a few wrapper functions unreferenced by design.
+  excludeShellChecks = [
+    "SC2086" # word splitting on purpose: env lists and profile triples
+    "SC2046" # same, command substitution into set --
+    "SC2329"
+  ];
+
+  runtimeDeps = [
+    curl
+    jq
+    coreutils
+    gawk
+    gnugrep
+    gnused
+    openssl
+  ];
+
+  lib = ''
+    ${builtins.readFile ./log.sh}
+    ${builtins.readFile ./state.sh}
+    ${builtins.readFile ./decide.sh}
+    ${builtins.readFile ./hcloud.sh}
+    ${builtins.readFile ./gitea.sh}
+  '';
+
+  handler = hectic.writeShellApplication {
+    inherit shell bashOptions;
+    inherit excludeShellChecks;
+    name = "gcr-webhook-handler";
+    runtimeInputs = [ socat ] ++ runtimeDeps;
+    text = ''
+      ${lib}
+      ${builtins.readFile ./webhook.sh}
+      gcr_state_init
+      gcr_handle_webhook || gcr_respond 500 "internal error"
+      exit 0
+    '';
+  };
+
+  webhook = hectic.writeShellApplication {
+    inherit shell bashOptions;
+    inherit excludeShellChecks;
+    name = "gitea-runner-webhook";
+    runtimeInputs = [ socat ];
+    text = ''
+      : "''${GCR_LISTEN_ADDR:=127.0.0.1}"
+      : "''${GCR_LISTEN_PORT:=8787}"
+      exec ${socat}/bin/socat -T5 -t5 \
+        "TCP-LISTEN:$GCR_LISTEN_PORT,bind=$GCR_LISTEN_ADDR,reuseaddr,fork" \
+        EXEC:"${handler}/bin/gcr-webhook-handler",pipes
+    '';
+  };
+
+  controller = hectic.writeShellApplication {
+    inherit shell bashOptions;
+    inherit excludeShellChecks;
+    name = "gitea-runner-controller";
+    runtimeInputs = runtimeDeps;
+    text = ''
+      ${lib}
+      ${builtins.readFile ./controller.sh}
+      gcr_main
+    '';
+  };
+in
+symlinkJoin {
+  name = "gitea-runner-controller";
+  paths = [
+    controller
+    webhook
+  ];
+}

package/gitea-runner-controller/gitea.sh

@@ -0,0 +1,33 @@
+#!/bin/dash
+# Gitea API wrappers for gitea-runner-controller.
+# Requires: GCR_GITEA_URL, GITEA_REGISTRATION_TOKEN_FILE, GITEA_ADMIN_TOKEN_FILE
+
+gcr_gitea_registration_token() {
+    token="$(gcr_gitea_admin_token)" || return 1
+    curl -fsS -X POST -H "Authorization: token $token" \
+        "$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners/registration-token" \
+        | jq -r '.token'
+}
+
+gcr_gitea_admin_token() {
+    test -r "${GITEA_ADMIN_TOKEN_FILE:-}" || {
+        gcr_log error --ns=gitea "GITEA_ADMIN_TOKEN_FILE missing"
+        return 1
+    }
+    tr -d '\n' < "$GITEA_ADMIN_TOKEN_FILE"
+}
+
+# gcr_gitea_list_runners — prints "id name" lines for org hectic-lab.
+gcr_gitea_list_runners() {
+    token="$(gcr_gitea_admin_token)" || return 1
+    curl -fsS -H "Authorization: token $token" \
+        "$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners?per_page=50" \
+        | jq -r '.entries[] | "\(.id) \(.name)"'
+}
+
+gcr_gitea_delete_runner() {
+    id="$1"
+    token="$(gcr_gitea_admin_token)" || return 1
+    curl -fsS -X DELETE -H "Authorization: token $token" \
+        "$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners/$id"
+}

package/gitea-runner-controller/hcloud.sh

@@ -0,0 +1,323 @@
+#!/bin/dash
+# Hetzner Cloud API wrappers for gitea-runner-controller.
+# Requires: HCLOUD_TOKEN_FILE, GCR_HETZNER_LOCATION, GCR_IMAGE_ID,
+#           GCR_ACT_RUNNER_VERSION, GCR_ACT_RUNNER_SHA256, GCR_NIX_VERSION,
+#           GCR_NIX_TARBALL_SHA256, GCR_GITEA_URL
+# All VMs carry the tag pair gitea-runner-controller=managed plus gcr.* metadata.
+
+GCR_API="https://api.hetzner.cloud/v1"
+
+gcr_hcloud_token() {
+    test -n "${HCLOUD_TOKEN_FILE:-}" && test -r "$HCLOUD_TOKEN_FILE" || {
+        gcr_log error --ns=hcloud "HCLOUD_TOKEN_FILE missing or unreadable"
+        return 1
+    }
+    tr -d '\n' < "$HCLOUD_TOKEN_FILE"
+}
+
+# All request state flows through files/exit codes, never command substitution
+# ($( ) runs in a subshell and would strand GCR_REQ_FAILED/GCR_LAST_HTTP).
+gcr_hcloud_req() {
+    # gcr_hcloud_req METHOD PATH [JSON_BODY]
+    # Body written to $GCR_LAST_BODY; exit 0 only on HTTP 2xx.
+    method="$1"; path="$2"; body="${3:-}"
+    token="$(gcr_hcloud_token)" || return 1
+    GCR_LAST_BODY="$(mktemp "${TMPDIR:-/tmp}/gcr-resp.XXXXXX")"
+    if [ -n "$body" ]; then
+        code="$(printf '%s' "$body" | curl -sS -X "$method" \
+            -H "Authorization: Bearer $token" \
+            -H 'Content-Type: application/json' \
+            --data-binary @- \
+            -o "$GCR_LAST_BODY" \
+            -w '%{http_code}' \
+            "$GCR_API$path")"
+    else
+        code="$(curl -sS -X "$method" \
+            -H "Authorization: Bearer $token" \
+            -o "$GCR_LAST_BODY" \
+            -w '%{http_code}' \
+            "$GCR_API$path")"
+    fi
+    case "$code" in 2??) return 0 ;; esac
+    gcr_log warn --ns=hcloud "request failed path=$path http=$code body=$(head -c 200 "$GCR_LAST_BODY" | gcr_redact)"
+    return 1
+}
+
+gcr_vm_list_managed() {
+    if gcr_hcloud_req GET "/servers?label_selector=gitea-runner-controller%3Dmanaged&per_page=50"; then
+        jq -S '.servers' "$GCR_LAST_BODY"
+    fi
+}
+
+gcr_vm_build_userdata() {
+    vm_name="$1"; label="$2"; reg_token="$3"
+
+    nix_conf='accept-flake-config = true
+experimental-features = nix-command flakes
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+sandbox = false'
+
+    runner_config="log:
+  level: info
+runner:
+  file: /var/lib/gitea-runner/.runner
+  capacity: 1
+  timeout: $(printf '%s' "$(gcr_label_profile "$label")" | awk '{print $2}')m
+  insecure: false
+  fetch_timeout: 5s
+  fetch_interval: 2s
+labels:
+  - \"$label:host\""
+
+    ssh_key_block=""
+    if [ -n "${GCR_DEBUG_SSH_PUBKEY:-}" ]; then
+        ssh_key_block="  - path: /root/.ssh/authorized_keys
+    permissions: '0600'
+    content: |
+      $GCR_DEBUG_SSH_PUBKEY"
+    fi
+
+    # NOTE(yukkop): token reaches only this VM's Hetzner metadata service;
+    # ephemeral registration makes it useless after the single job exits.
+    printf '%s' "#cloud-config
+write_files:
+$ssh_key_block
+  - path: /etc/ssh/sshd_config.d/99-gcr-root.conf
+    permissions: '0644'
+    content: |
+      PermitRootLogin prohibit-password
+      PubkeyAuthentication yes
+  - path: /etc/nix/nix.conf
+    content: |
+$(printf '%s\n' "$nix_conf" | sed 's/^/      /')
+  - path: /etc/gitea-runner/config.yaml
+    content: |
+$(printf '%s\n' "$runner_config" | sed 's/^/      /')
+  - path: /etc/systemd/system/gitea-runner.service
+    content: |
+      [Unit]
+      Description=Gitea ephemeral Actions runner
+      After=network-online.target gcr-bootstrap.service
+      Requires=gcr-bootstrap.service
+
+      [Service]
+      Type=simple
+      Environment=GITEA_INSTANCE_URL=$GCR_GITEA_URL
+      Environment=GITEA_RUNNER_REGISTRATION_TOKEN=$reg_token
+      ExecStart=/usr/local/bin/act_runner daemon --ephemeral --config /etc/gitea-runner/config.yaml
+      Restart=on-failure
+      RestartSec=5
+
+      [Install]
+      WantedBy=multi-user.target
+  - path: /etc/systemd/system/gcr-bootstrap.service
+    content: |
+      [Unit]
+      Description=Bootstrap Nix + act_runner for ephemeral CI job
+      After=network-online.target
+      Wants=network-online.target
+      Before=gitea-runner.service
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=true
+      ExecStart=/usr/local/sbin/gcr-bootstrap
+
+      [Install]
+      WantedBy=multi-user.target
+  - path: /usr/local/sbin/gcr-bootstrap
+    permissions: '0700'
+    content: |
+      #!/bin/sh
+      set -eu
+      exec > /var/log/gcr-bootstrap.log 2>&1
+      curl -fsSL \"https://nixos.org/releases/nix/$GCR_NIX_VERSION/nix-$GCR_NIX_VERSION-x86_64-linux.tar.xz\" -o /tmp/nix.tar.xz
+      printf '%s  /tmp/nix.tar.xz\n' \"$GCR_NIX_TARBALL_SHA256\" | sha256sum -c -
+      tar -xJf /tmp/nix.tar.xz -C /tmp
+      /tmp/nix-$GCR_NIX_VERSION-x86_64-linux/install --no-daemon
+      rm -rf /tmp/nix*
+      curl -fsSL \"https://gitea.com/gitea/act_runner/releases/download/$GCR_ACT_RUNNER_VERSION/act_runner-\$(printf '%s' \"$GCR_ACT_RUNNER_VERSION\" | sed 's/^v//')-linux-amd64\" -o /usr/local/bin/act_runner
+      printf '%s  /usr/local/bin/act_runner\n' \"$GCR_ACT_RUNNER_SHA256\" | sha256sum -c -
+      chmod 0755 /usr/local/bin/act_runner
+      mkdir -p /var/lib/gitea-runner
+runcmd:
+  - [ sh, -c, 'systemctl enable --now sshd.service 2>/dev/null || systemctl enable --now ssh 2>/dev/null || true' ]
+  - [ sh, -c, 'systemctl restart sshd.service 2>/dev/null || systemctl restart ssh 2>/dev/null || true' ]
+  - [ systemctl, enable, --now, gitea-runner.service ]
+"
+}
+
+# gcr_vm_create NAME LABEL SERVER_TYPE TTL_MIN REG_TOKEN JOB_ID ATTEMPT REPO
+# Prints new server id.
+gcr_vm_create() {
+    vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4"
+    reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8"
+
+    test -n "${GCR_IMAGE_ID:-}" || {
+        gcr_log error --ns=hcloud "GCR_IMAGE_ID not set; refusing VM creation"
+        return 1
+    }
+
+    userdata="$(gcr_vm_build_userdata "$vm_name" "$label" "$reg_token")"
+    payload="$(jq -n \
+        --arg name "$vm_name" \
+        --arg stype "$server_type" \
+        --arg image "$GCR_IMAGE_ID" \
+        --arg loc "${GCR_HETZNER_LOCATION:-nbg1}" \
+        --arg udata "$userdata" \
+        --arg jid "$job_id" \
+        --arg att "$attempt" \
+        --arg repo "$repo" \
+        --arg label "$label" \
+        --arg ts "$(date -u '+%s')" \
+        --arg ttl "$ttl_min" \
+        --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \
+        '{name:$name, server_type:$stype, image:$image, location:$loc,
+          start_after_create:true,
+          labels:{
+            "gitea-runner-controller":"managed",
+            "gcr.job-id":$jid, "gcr.run-attempt":$att,
+            "gcr.repo":$repo_safe, "gcr.label":$label,
+            "gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')"
+
+    # Hetzner placement is occasionally transient (resource_unavailable);
+    # retry a few times before giving up. NOTE: userdata/cloud-init is NOT
+    # used — bootstrap happens over SSH from the controller (see
+    # gcr_vm_bootstrap_ssh); MicroOS snapshot's Hetzner datasource cannot
+    # fetch user-data (DHCP Exception on this image lineage).
+    attempt_n=0
+    while :; do
+        attempt_n=$((attempt_n + 1))
+        if gcr_hcloud_req POST /servers "$payload"; then
+            jq -r '.server.id' "$GCR_LAST_BODY"
+            return 0
+        fi
+        gcr_log warn --ns=hcloud "create attempt=$attempt_n failed"
+        [ "$attempt_n" -ge 3 ] && return 1
+        sleep $((attempt_n * 10))
+    done
+}
+
+# gcr_vm_destroy SERVER_ID — idempotent best-effort destroy.
+gcr_vm_destroy() {
+    if ! gcr_hcloud_req DELETE "/servers/$1"; then
+        gcr_log warn --ns=hcloud "destroy failed or already gone: server $1"
+        return 1
+    fi
+}
+
+# Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's
+# cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the
+# controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose
+# public half is authorized on every ephemeral VM (project ssh-key injection).
+gcr_bootstrap_script() {
+    # gcr_bootstrap_script LABEL REG_TOKEN TTL_MIN RUNNER_NAME -> POSIX sh payload
+    label="$1"; reg_token="$2"; ttl_min="$3"; runner_name="$4"
+    nix_conf='accept-flake-config = true
+experimental-features = nix-command flakes
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+sandbox = false'
+
+    runner_config="log:
+  level: info
+runner:
+  file: /var/lib/gitea-runner/.runner
+  capacity: 1
+  timeout: ${ttl_min}m
+  insecure: false
+  fetch_timeout: 5s
+  fetch_interval: 2s
+labels:
+  - \"$label:host\""
+
+    cat <<BSEOF
+exec >/var/log/gcr-bootstrap.log 2>&1
+set -eu
+mkdir -p /etc/nix /etc/gitea-runner /var/lib/gitea-runner /usr/local/bin
+cat > /etc/nix/nix.conf <<'NIXEOF'
+$nix_conf
+NIXEOF
+cat > /etc/gitea-runner/config.yaml <<'CFGEOF'
+$runner_config
+CFGEOF
+cat > /usr/local/sbin/gcr-runner-start <<STARTEOF
+#!/bin/sh
+set -eu
+if [ ! -f /var/lib/gitea-runner/.runner ]; then
+  /usr/local/bin/act_runner register --no-interactive --instance $GCR_GITEA_URL --token $reg_token --name $runner_name --labels $label:host --config /etc/gitea-runner/config.yaml
+fi
+exec /usr/local/bin/act_runner daemon --config /etc/gitea-runner/config.yaml
+STARTEOF
+chmod 0700 /usr/local/sbin/gcr-runner-start
+cat > /etc/systemd/system/gitea-runner.service <<UNITEOF
+[Unit]
+Description=Gitea ephemeral Actions runner
+After=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=/var/lib/gitea-runner
+Environment=HOME=/var/lib/gitea-runner
+ExecStart=/usr/local/sbin/gcr-runner-start
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+UNITEOF
+cat > /usr/local/sbin/gcr-install <<INSEOF
+#!/bin/sh
+set -eu
+if [ "$label" = "nix" ]; then
+  curl -fsSL "https://releases.nixos.org/nix/nix-$GCR_NIX_VERSION/nix-$GCR_NIX_VERSION-x86_64-linux.tar.xz" -o /tmp/nix.tar.xz
+  printf '%s  /tmp/nix.tar.xz\n' "$GCR_NIX_TARBALL_SHA256" | sha256sum -c -
+  tar -xJf /tmp/nix.tar.xz -C /tmp
+  /tmp/nix-$GCR_NIX_VERSION-x86_64-linux/install --no-daemon
+  rm -rf /tmp/nix*
+fi
+curl -fsSL "https://gitea.com/gitea/act_runner/releases/download/$GCR_ACT_RUNNER_VERSION/act_runner-\$(printf '%s' "$GCR_ACT_RUNNER_VERSION" | sed 's/^v//')-linux-amd64" -o /usr/local/bin/act_runner
+printf '%s  /usr/local/bin/act_runner\n' "$GCR_ACT_RUNNER_SHA256" | sha256sum -c -
+chmod 0755 /usr/local/bin/act_runner
+INSEOF
+chmod 0700 /usr/local/sbin/gcr-install
+/usr/local/sbin/gcr-install
+systemctl daemon-reload
+systemctl enable --now gitea-runner.service
+BSEOF
+}
+
+# gcr_vm_bootstrap_ssh IP LABEL REG_TOKEN — blocking; returns ssh exit status.
+gcr_vm_bootstrap_ssh() {
+    ip="$1"; label="$2"; reg_token="$3"; runner_name="$4"
+    test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || {
+        gcr_log error --ns=hcloud "GCR_SSH_PRIVKEY_FILE missing or unreadable"
+        return 1
+    }
+    key_tmp="$(mktemp "${TMPDIR:-/tmp}/gcr-sshkey.XXXXXX")"
+    cat "$GCR_SSH_PRIVKEY_FILE" > "$key_tmp"
+    printf '\n' >> "$key_tmp"
+    chmod 0600 "$key_tmp"
+    SSH_OPTS="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes"
+
+    waited=0
+    until ssh $SSH_OPTS "root@$ip" true 2>/dev/null; do
+        waited=$((waited + 5))
+        [ "$waited" -ge 300 ] && {
+            gcr_log warn --ns=hcloud "sshd never came up on $ip"
+            rm -f "$key_tmp"
+            return 1
+        }
+        sleep 5
+    done
+
+    ttl_min="$(printf '%s' "$(gcr_label_profile "$label")" | awk '{print $2}')"
+    script="$(gcr_bootstrap_script "$label" "$reg_token" "$ttl_min" "$runner_name")"
+    if printf '%s' "$script" | ssh $SSH_OPTS "root@$ip" sh -s; then
+        rm -f "$key_tmp"
+        return 0
+    fi
+    rm -f "$key_tmp"
+    return 1
+}

package/gitea-runner-controller/log.sh

@@ -0,0 +1,77 @@
+#!/bin/dash
+# Log helper for gitea-runner-controller.
+# Verbosity via GCR_LOG env: "<level>[;<ns>=<level>]..." e.g. "info;alloc=debug".
+
+MAGENTA='\033[0;35m'
+BLUE='\033[0;34m'
+GREEN='\033[0;32m'
+CYAN='\033[0;36m'
+YELLOW='\033[0;33m'
+RED='\033[0;31m'
+WHITE='\033[0;37m'
+NC='\033[0m'
+
+gcr_log_enabled() {
+    level="$1"
+    ns="${2:-core}"
+    conf="${GCR_LOG:-info}"
+    ns_level=""
+    default_level=""
+    oldIFS="$IFS"
+    IFS=';'
+    for pair in $conf; do
+        case "$pair" in
+            *=*) ns_name="${pair%%=*}"; ns_level="${pair#*=}"
+                 [ "$ns_name" = "$ns" ] && { IFS="$oldIFS"; echo "$ns_level"; return 0; } ;;
+            *)   default_level="$pair" ;;
+        esac
+    done
+    IFS="$oldIFS"
+    echo "${default_level:-info}"
+}
+
+gcr_log_level_rank() {
+    case "$1" in
+        trace)  echo 0 ;;
+        debug)  echo 1 ;;
+        info)   echo 2 ;;
+        notice) echo 3 ;;
+        warn)   echo 4 ;;
+        error)  echo 5 ;;
+        panic)  echo 6 ;;
+        *)      echo 7 ;;
+    esac
+}
+
+gcr_log() {
+    level="$1"; shift
+    ns="core"
+    case "$1" in
+        --ns=*) ns="${1#--ns=}"; shift ;;
+    esac
+
+    want="$(gcr_log_enabled "$level" "$ns")"
+    [ "$(gcr_log_level_rank "$level")" -ge "$(gcr_log_level_rank "$want")" ] || return 0
+
+    ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+    msg="$1"
+    case "$level" in
+        trace)  color="$WHITE"   ;;
+        debug)  color="$BLUE"    ;;
+        info)   color="$GREEN"   ;;
+        notice) color="$CYAN"    ;;
+        warn)   color="$YELLOW"  ;;
+        error)  color="$RED"     ;;
+        panic)  color="$MAGENTA" ;;
+        *)      color="$WHITE"   ;;
+    esac
+    printf '%b\n' "${color}${ts} ${level}[${ns}]${NC} $msg" >&2
+}
+
+# Security: keep credential values out of logs; keys containing TOKEN/SECRET
+# and Authorization token headers are rewritten to <redacted>.
+gcr_redact() {
+    sed -E \
+        -e 's/([A-Za-z0-9_]*(TOKEN|SECRET)[A-Za-z0-9_]*)(=|: ?"?)[^ "]+/\1\3<redacted>/g' \
+        -e 's/(Authorization: *token )[^ ]+/\1<redacted>/g'
+}

package/gitea-runner-controller/state.sh

@@ -0,0 +1,74 @@
+#!/bin/dash
+# State primitives for gitea-runner-controller.
+# Layout:
+#   $GCR_STATE_DIR/jobs/<job_id>.json   allocation records
+#   $GCR_STATE_DIR/jobs/.lock.<key>/    mkdir(2) atomicity guards
+#   $GCR_STATE_DIR/events.jsonl         append-only audit
+#   $GCR_STATE_DIR/budget/<YYYY-MM>     estimated EUR spent this month
+
+gcr_state_init() {
+    test -n "${GCR_STATE_DIR:-}" || { echo "GCR_STATE_DIR is not set" >&2; return 1; }
+    mkdir -p "$GCR_STATE_DIR/jobs" "$GCR_STATE_DIR/budget"
+    touch "$GCR_STATE_DIR/events.jsonl"
+}
+
+gcr_alloc_key() {
+    printf '%s-%s' "$1" "$2"
+}
+
+gcr_record_path() {
+    printf '%s/jobs/%s.json' "$GCR_STATE_DIR" "$(gcr_alloc_key "$1" "$2")"
+}
+
+# mkdir(2) atomicity guard: succeeds exactly once per key until released.
+gcr_lock_acquire() {
+    mkdir "$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$1")" 2>/dev/null
+}
+
+gcr_lock_release() {
+    rm -rf "$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$1")"
+}
+
+gcr_record_get() {
+    # Missing record is a normal answer, not an error; must not trip errexit.
+    { cat "$(gcr_record_path "$1" "$2")" 2>/dev/null || true; }
+}
+
+# mktemp+mv keeps concurrent readers away from partially written records.
+gcr_record_put() {
+    tmp="$(mktemp "$(dirname "$(gcr_record_path "$1" "$2")")/.tmp.XXXXXX")"
+    printf '%s\n' "$3" > "$tmp"
+    mv -f "$tmp" "$(gcr_record_path "$1" "$2")"
+}
+
+gcr_record_del() {
+    rm -f "$(gcr_record_path "$1" "$2")"
+}
+
+gcr_record_field() {
+    printf '%s' "$1" | jq -r --arg f "$2" '.[$f] // ""'
+}
+
+gcr_event() {
+    printf '{"ts":"%s","event":"%s","job_id":"%s","detail":%s}\n' \
+        "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$1" "$2" \
+        "$(printf '%s' "$3" | jq -Rs .)" >> "$GCR_STATE_DIR/events.jsonl"
+}
+
+# Exit-code contract: 0 = recorded under budget, 1 = would exceed cap.
+gcr_budget_add() {
+    rate="$1"; ttl_min="$2"
+    month="$(date -u '+%Y-%m')"
+    file="$GCR_STATE_DIR/budget/$month"
+    current="$(cat "$file" 2>/dev/null || echo 0)"
+    projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')"
+    if awk -v p="$projected" -v b="${GCR_BUDGET_EUR_MONTHLY:-15}" 'BEGIN {exit !(p > b)}'; then
+        return 1
+    fi
+    printf '%s\n' "$projected" > "$file"
+    return 0
+}
+
+gcr_active_records() {
+    grep -l '"status":"\(pending_vm\|vm_active\|deferred\)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
+}

package/gitea-runner-controller/webhook.sh

@@ -0,0 +1,233 @@
+#!/bin/dash
+# One-shot HTTP handler for Gitea workflow_job webhooks.
+# Invoked per connection by socat; request on stdin, response on stdout.
+
+RESPONSE_CODE=204
+RESPONSE_BODY=""
+
+gcr_respond() {
+    code="$1"; body="${2:-}"
+    reason=""
+    case "$code" in
+        200) reason="OK" ;;
+        202) reason="Accepted" ;;
+        204) reason="No Content" ;;
+        400) reason="Bad Request" ;;
+        403) reason="Forbidden" ;;
+        413) reason="Payload Too Large" ;;
+        *)   reason="No Content" ;;
+    esac
+    printf 'HTTP/1.1 %s %s\r\n' "$code" "$reason"
+    printf 'Content-Type: text/plain\r\n'
+    printf 'Content-Length: %s\r\n' "$(printf '%s' "$body" | wc -c)"
+    printf 'Connection: close\r\n\r\n'
+    [ -n "$body" ] && printf '%s\n' "$body"
+}
+
+gcr_read_request() {
+    request_line=""
+    gcr_hdr_event_type=""
+    gcr_hdr_delivery=""
+    gcr_hdr_signature=""
+    content_length=0
+
+    # Correctness depends on dash reading stdin byte-by-byte (no lookahead
+    # buffer); body bytes must remain unconsumed for `head -c` below.
+    IFS= read -r request_line || return 1
+
+    while :; do
+        IFS= read -r line || break
+        line="$(printf '%s' "$line" | tr -d '\r')"
+        [ -z "$line" ] && break
+        name="$(printf '%s' "$line" | cut -d: -f1 | tr '[:upper:]' '[:lower:]')"
+        value="$(printf '%s' "${line#*:}" | sed 's/^ *//')"
+        case "$name" in
+            x-gitea-event-type) gcr_hdr_event_type="$value" ;;
+            x-gitea-delivery)   gcr_hdr_delivery="$value" ;;
+            x-gitea-signature)  gcr_hdr_signature="$value" ;;
+            content-length)     content_length="$value" ;;
+        esac
+    done
+
+    case "$request_line" in
+        "POST "*" HTTP/"*) ;;
+        *) return 1 ;;
+    esac
+
+    case "$content_length" in
+        ''|*[!0-9]*) return 1 ;;
+    esac
+    [ "$content_length" -le 65536 ] || { gcr_respond 413 "payload too large"; exit 0; }
+
+    gcr_body="$(head -c "$content_length")"
+}
+
+gcr_verify_signature() {
+    test -r "${GITEA_WEBHOOK_SECRET_FILE:-}" || {
+        gcr_log error --ns=webhook "GITEA_WEBHOOK_SECRET_FILE missing"
+        return 1
+    }
+    secret="$(cat "$GITEA_WEBHOOK_SECRET_FILE")"
+    expected="$(printf '%s' "$gcr_body" \
+        | openssl dgst -sha256 -hmac "$secret" -hex \
+        | awk '{print $NF}')"
+    # NOTE(yukkop): shell string compare is not constant-time; acceptable here
+    # because the secret is high-entropy and bodies are signed, not encrypted.
+    [ "$expected" = "$gcr_hdr_signature" ]
+}
+
+gcr_alloc() {
+    job_id="$1"; attempt="$2"; repo="$3"; labels_json="$4"
+
+    label="$(printf '%s' "$labels_json" | jq -r '.[0] // ""')"
+    label_count="$(printf '%s' "$labels_json" | jq 'length')"
+
+    existing="$(gcr_record_get "$job_id" "$attempt")"
+    if [ -n "$existing" ]; then
+        gcr_log debug --ns=alloc "duplicate delivery for $(gcr_alloc_key "$job_id" "$attempt")"
+        RESPONSE_CODE=204
+        return 0
+    fi
+
+    key="$(gcr_alloc_key "$job_id" "$attempt")"
+    if ! gcr_lock_acquire "$key"; then
+        RESPONSE_CODE=204
+        return 0
+    fi
+
+    if [ "$label_count" -ne 1 ]; then
+        gcr_lock_release "$key"
+        gcr_event "refused" "$job_id" "{\"repo\":\"$repo\",\"label_count\":$label_count}"
+        RESPONSE_CODE=202; RESPONSE_BODY="refused: exactly one label required"
+        return 0
+    fi
+
+    if ! profile="$(gcr_decide "$label" "$repo")"; then
+        gcr_lock_release "$key"
+        gcr_event "refused" "$job_id" "{\"repo\":\"$repo\",\"label\":\"$label\"}"
+        RESPONSE_CODE=202; RESPONSE_BODY="refused: repo or label not allowed"
+        return 0
+    fi
+
+    set -- $profile
+    server_type="$1"; ttl_min="$2"; rate="$3"
+
+    active="$(gcr_count_active)"
+    repo_active="$(gcr_count_active_repo "$repo")"
+    if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \
+        || [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then
+        rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
+            --arg l "$label" --arg t "$(date -u '+%s')" \
+            '{job_id:$j, run_attempt:$a, repo:$r, label:$l,
+              created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
+              status:"deferred"}')"
+        gcr_record_put "$job_id" "$attempt" "$rec"
+        gcr_lock_release "$key"
+        gcr_event "deferred" "$job_id" "{\"active\":$active,\"repo_active\":$repo_active}"
+        RESPONSE_CODE=202; RESPONSE_BODY="deferred: capacity"
+        return 0
+    fi
+
+    if ! gcr_budget_add "$rate" "$ttl_min"; then
+        gcr_record_del "$job_id" "$attempt"
+        gcr_lock_release "$key"
+        gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}"
+        RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted"
+        return 0
+    fi
+
+    reg_token="$(gcr_gitea_registration_token)" || {
+        gcr_record_del "$job_id" "$attempt"
+        gcr_lock_release "$key"
+        gcr_event "token-error" "$job_id" "{}"
+        RESPONSE_CODE=202; RESPONSE_BODY="registration token unavailable"
+        return 0
+    }
+
+    vm_name="gcr-${job_id}-${attempt}"
+    vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
+        "$reg_token" "$job_id" "$attempt" "$repo")" || {
+        gcr_record_del "$job_id" "$attempt"
+        gcr_lock_release "$key"
+        gcr_event "vm-create-failed" "$job_id" "{}"
+        RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
+        return 0
+    }
+
+    rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
+        --arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \
+        --arg vn "$vm_name" --arg ttl "$ttl_min" \
+        '{job_id:$j, run_attempt:$a, repo:$r, label:$l,
+          created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
+          vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
+    gcr_record_put "$job_id" "$attempt" "$rec"
+    gcr_lock_release "$key"
+    gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min}"
+
+    RESPONSE_CODE=202; RESPONSE_BODY="allocated $vm_name"
+}
+
+gcr_deallocate() {
+    job_id="$1"; attempt="$2"; new_status="$3"
+
+    rec="$(gcr_record_get "$job_id" "$attempt")"
+    [ -n "$rec" ] || return 0
+
+    vm_id="$(gcr_record_field "$rec" vm_id)"
+    if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
+        gcr_vm_destroy "$vm_id" || true
+        gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
+    fi
+
+    gcr_record_del "$job_id" "$attempt"
+    gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
+}
+
+gcr_handle_webhook() {
+    gcr_read_request || { gcr_respond 400 ""; exit 0; }
+
+    [ "$gcr_hdr_event_type" = "workflow_job" ] || {
+        gcr_log debug --ns=webhook "ignored event type: $gcr_hdr_event_type"
+        gcr_respond 204 ""; exit 0
+    }
+
+    gcr_verify_signature || {
+        gcr_log warn --ns=webhook "invalid signature, delivery=$gcr_hdr_delivery"
+        gcr_respond 403 "invalid signature"; exit 0
+    }
+
+    action="$(printf '%s' "$gcr_body" | jq -r '.action // ""')"
+    job_id="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.id // ""')"
+    attempt="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.run_attempt // ""')"
+    repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')"
+    labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')"
+
+    case "$action:$job_id" in
+        :*|"queued:"|*":0") gcr_respond 400 "malformed payload"; exit 0 ;;
+    esac
+
+    case "$action" in
+        queued)
+            gcr_alloc "$job_id" "$attempt" "$repo" "$labels_json"
+            gcr_log info --ns=alloc "queued job=$job_id repo=$repo code=$RESPONSE_CODE $RESPONSE_BODY"
+            ;;
+        in_progress)
+            rec="$(gcr_record_get "$job_id" "$attempt")"
+            if [ -n "$rec" ]; then
+                rec="$(printf '%s' "$rec" | jq -c '.status = "vm_active"')"
+                gcr_record_put "$job_id" "$attempt" "$rec"
+            fi
+            RESPONSE_CODE=204
+            ;;
+        completed)
+            gcr_deallocate "$job_id" "$attempt" "completed"
+            RESPONSE_CODE=204
+            ;;
+        *)
+            gcr_log debug --ns=webhook "unhandled action: $action"
+            RESPONSE_CODE=204
+            ;;
+    esac
+
+    gcr_respond "$RESPONSE_CODE" "$RESPONSE_BODY"
+}

sus/gitea-runners.yaml

@@ -1,6 +1,11 @@
 gitea:
     hectic-lab:
         org-runner-registration-token: ENC[AES256_GCM,data:Hez1q44P/gxRfpA5Nl1bsf5Be719z/DadWyJMoghaeh+7dX3TqhFmw==,iv:ufme+uMcsI8pw5DcAlxJJKFbcj7XSg2E4MfC12U3sO0=,tag:JLyyBb5PTLY02jOGeB7EEg==,type:str]
+        controller:
+            hcloud-token: ENC[AES256_GCM,data:mw2P6v1k1MOtIpuARG7++T3ica8gY7HdfGOlzr9DyMzqDl7hGAYmpLitVDu++5mZGHZoVboH9YzVwU9d6frGhg==,iv:db2Dy8fZ3vaOlvT5QAxptBY/h2glIroPQq2gpQVIno0=,tag:haR1YKTegFQytkQ6ygAKgw==,type:str]
+            webhook-secret: ENC[AES256_GCM,data:NppDTo0T+DDlCaNp6mcQWgGfaygFGwDR5bd6oylxFIk/XhSz6ATUpNWSrn9oZm2geuc/BHgGAwcgQU1LnhnJGw==,iv:zhwOxR6Pf/NKSiDME/ipNBhcDize/t4WcSob13QmjsA=,tag:FXJflbfmpgqKZuiEYwBcrg==,type:str]
+            admin-token: ENC[AES256_GCM,data:inIN+ISgS+HgCEu/r9lfjRbbbCz83NCcwoSaeeqJys6JocEGBy0law==,iv:Lj6f3flrQWatHHy9diMQQmFsnSU0ckqoZB0AX6eL00U=,tag:OazXKYMPbYsy+GE2eVCaNQ==,type:str]
+            ssh-private-key: ENC[AES256_GCM,data:+RH7sfrKpFo+lVgTunc+qWrqKgbwWHxRDt5iXySLQCeNQR/Qr/S9NWnJgDA4iwZqXxeDq/WlMpKMLfPu7lGeVuMr0sp3D9BCljtRWeFp3/OsWeOaDjHjIxiQF+E6XeaU52lwDaBb3xTUV+4iWZVYHytEtaKkSplxDkNqa+AOxlJ6q5xK1KALJBajR6AhTiLt/IrSDrBCP2Sd79wiBaulcD+WOem25mm8SzbSbnVZ1NnSjvh5O6pKKmRho9yjPVj9yXPad3Hpder986JZSH/wte9koNAaBzEjtAPZd4Do00xboV0PMgFm2aTft2nbDbnpxDMN0wNbcpOmOmGzraxRRTFs8o/UHRsDnWw8ZZGTiqqivCt9q0cB5mVJSxaQpJlzqCX3hjPz8sbCdw/vzIQc76fQHqHcleeb1oht+2E6T2OdRStEDI2u1ts+yF0d2gk/99wYaMK4Oun7qCyffHKsFt/vb1tAT3IXNW0zgkYWJhBd3oUsbV16ImL82VZSDyjIwBFwORkOU6PIUWuqVIo=,iv:k4dQC5FTc1k01z33gVXPTna63l+Ixr1aW7/OXGt7R/A=,tag:/dRO7wRjyYTR3p+6H17w+Q==,type:str]
 hetzner:
     hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str]
 s3:
@@ -53,7 +58,7 @@ sops:
             UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P
             MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-24T17:35:02Z"
+    lastmodified: "2026-08-27T07:26:16Z"
-    mac: ENC[AES256_GCM,data:Nulehy1TdIx4bGCRLWR6JYKqiyadelKG1fuknfPXArjUGzr4q6Pa+SgvBaKL8zcIlu224SxHNUSo3EfPj/O+f5xPH95hhD1vpbZbBBNsls5gMeVMhSELyfhJUn6e7zSwRGzMrY8wZ4VkBRsitHmMTC+Qg33YXsLEm79GKIhzcKI=,iv:zAhpjVhuMsFN/LHmAUGP2/FpsSlBs++bmkktLBxXQnI=,tag:B0wp4ZWfZ6/+ZyBuaKAQHw==,type:str]
+    mac: ENC[AES256_GCM,data:H71K/HjlVCdaOWLcOsdHJo591aCr5+Fo5ObX2H7yNJAOaN5sE1HH/blz9koxivFEyr/yCr6a9cMZzzQoiZ1M4iFLwjiH/ZkLm44tZJsv9Dc+IJskNO+4Am+31OEgMQaf8ciVSJqn5PRZqPV0bg+OxznShxY/ClxCeIxNINGsvnw=,iv:+FK69Dm5XnPsu4CX+MFILA1Ag+SGJNBV0g1TiT7snRI=,tag:5OX0VVTipocpfjZZ+Q+pLA==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1