From c5026d3e9d083822bc6cd7c80aa912f48c8b04d5 Mon Sep 17 00:00:00 2001 From: yukkop Date: Thu, 27 Aug 2026 12:34:23 +0000 Subject: [PATCH] feat: zero-idle gite runners infra --- infra/gitea-runners/k8s/statefulset.yaml | 3 +- infra/gitea-runners/runbook.md | 114 +++++++ .../service/gitea-runner-controller.nix | 230 +++++++++++++ nixos/system/hectic-lab/hectic-lab.nix | 17 +- package/default.nix | 1 + package/gitea-runner-controller/controller.sh | 219 ++++++++++++ package/gitea-runner-controller/decide.sh | 70 ++++ package/gitea-runner-controller/default.nix | 92 +++++ package/gitea-runner-controller/gitea.sh | 33 ++ package/gitea-runner-controller/hcloud.sh | 323 ++++++++++++++++++ package/gitea-runner-controller/log.sh | 77 +++++ package/gitea-runner-controller/state.sh | 74 ++++ package/gitea-runner-controller/webhook.sh | 233 +++++++++++++ sus/gitea-runners.yaml | 9 +- 14 files changed, 1491 insertions(+), 4 deletions(-) create mode 100644 nixos/module/hectic/service/gitea-runner-controller.nix create mode 100644 package/gitea-runner-controller/controller.sh create mode 100644 package/gitea-runner-controller/decide.sh create mode 100644 package/gitea-runner-controller/default.nix create mode 100644 package/gitea-runner-controller/gitea.sh create mode 100644 package/gitea-runner-controller/hcloud.sh create mode 100644 package/gitea-runner-controller/log.sh create mode 100644 package/gitea-runner-controller/state.sh create mode 100644 package/gitea-runner-controller/webhook.sh diff --git a/infra/gitea-runners/k8s/statefulset.yaml b/infra/gitea-runners/k8s/statefulset.yaml index 81e330d5..61b9bd31 100644 --- a/infra/gitea-runners/k8s/statefulset.yaml +++ b/infra/gitea-runners/k8s/statefulset.yaml @@ -8,7 +8,8 @@ metadata: app.kubernetes.io/part-of: gitea-actions spec: serviceName: gitea-runner - replicas: 1 + # Rollback-only pool. Keep manifest present, default replicas 0. + replicas: 0 podManagementPolicy: Parallel selector: matchLabels: diff --git a/infra/gitea-runners/runbook.md b/infra/gitea-runners/runbook.md index 50249ff5..d6d652d1 100644 --- a/infra/gitea-runners/runbook.md +++ b/infra/gitea-runners/runbook.md @@ -501,3 +501,117 @@ Do not release unless the following evidence files exist and are readable: If any evidence file is missing, stop and collect it before treating the runbook as complete. + +## Ephemeral VM runner cutover + +This section governs replacing the fixed K8s runner pool with the +ephemeral-VM controller (`package/gitea-runner-controller`) on this host. +The K8s pool above remains rollback-only until cutover is explicitly accepted. + +### Operator gates (all three required before enable) + +1. Secrets — add to `sus/gitea-runners.yaml` under + `gitea/hectic-lab/controller/*`: + + ```sh + sops sus/gitea-runners.yaml + # add keys: + # gitea: + # hectic-lab: + # controller: + # hcloud-token: + # webhook-secret: + # admin-token: + ``` + + The org registration token key + `gitea/hectic-lab/org-runner-registration-token` already exists and is reused. + +2. Base image — build the MicroOS snapshot and record its id: + + ```sh + nix develop .#gitea-runners -c gitea-runners-build-microos-snapshots x86 + hcloud image list --selector '' -o json | jq '.[] | select(.type=="snapshot")' + ``` + +3. DNS — A record `runners.hectic-lab.com -> 128.140.75.58` (ACME needs it). + +### Enable + +```sh +# nixos/system/hectic-lab/hectic-lab.nix: resolve the FIXME block +hectic.services.gitea-runner-controller = { + enable = true; + imageId = ""; +}; +nixos-rebuild --target root@128.140.75.58 switch +systemctl status gitea-runner-controller.service gitea-runner-webhook.service +``` + +### Register the Gitea webhook + +Org-level (preferred) or per-repo, on `https://gitea.hectic-lab.com`: + +- URL: `https://runners.hectic-lab.com/` +- Method: `POST`, content type: JSON +- Secret: value of `gitea/hectic-lab/controller/webhook-secret` +- Trigger events: `Workflow jobs` only (`workflow_job`) + +### Pre-flight verification (before first real job) + +```sh +curl -fsS https://runners.hectic-lab.com/ -o /dev/null -w '%{http_code}\n' # any 4xx from handler = reachable +journalctl -u gitea-runner-webhook -n 20 --no-pager +hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed")] | length' # expect 0 +``` + +Zero managed VMs at idle is the steady-state assertion. + +### End-to-end acceptance (Task 9) + +Trigger `.gitea/workflows/runner-nix-smoke.yaml` via workflow_dispatch, then: + +```sh +watch_labels() { hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed") | {id,name,labels}]'; } +watch_labels # exactly one VM while queued/running +journalctl -f -u gitea-runner-controller # vm-created / vm-destroyed events +watch_labels # expect [] after completion +curl -fsS -H "Authorization: token $ADMIN" \ + https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners \ + | jq '[.entries[] | select(.name | startswith("gcr-"))] | length' # expect 0 +``` + +Failure paths to verify identically: duplicate delivery (send same webhook twice +via Gitea UI "Test delivery"), cancelled job, unknown-label job. + +### Rollback + +K8s rollback pool now defaults to deleted state: + +- `infra/gitea-runners/k8s/statefulset.yaml` keeps `replicas: 0` +- old kube-hetzner nodes may be deleted to preserve zero idle cost +- PVCs and IaC remain for manual rollback only + +Re-enable sequence: + +```sh +# 1. stop ephemeral path +sed -i 's/hectic.services.gitea-runner-controller = {.*}/\/* disabled *\//' \ + nixos/system/hectic-lab/hectic-lab.nix # or set enable = false +nixos-rebuild --target root@128.140.75.58 switch + +# 2. reprovision old kube-hetzner nodes when they were deleted: +tofu -chdir=infra/gitea-runners/opentofu apply + +# 3. restore kubeconfig / cluster access, then re-enable K8s runner pool: +kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5 +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +``` + +Any surviving ephemeral VMs after step 1 must be destroyed manually once: + +```sh +hcloud server list -o json \ + | jq -r '.[] | select(.labels["gitea-runner-controller"]=="managed") | .id' \ + | xargs -r -n1 hcloud server delete +``` diff --git a/nixos/module/hectic/service/gitea-runner-controller.nix b/nixos/module/hectic/service/gitea-runner-controller.nix new file mode 100644 index 00000000..036da924 --- /dev/null +++ b/nixos/module/hectic/service/gitea-runner-controller.nix @@ -0,0 +1,230 @@ +{ + inputs, + flake, + self, +}: +{ + pkgs, + lib, + config, + ... +}: +let + system = pkgs.stdenv.hostPlatform.system; + cfg = config.hectic.services.gitea-runner-controller; + + secretPrefix = "gitea-runner-controller"; + mkControllerSecret = name: { + name = "${secretPrefix}/${name}"; + value = { + sopsFile = flake + "/sus/gitea-runners.yaml"; + key = "gitea/hectic-lab/controller/${name}"; + }; + }; + + controllerSecrets = builtins.listToAttrs (map mkControllerSecret [ + "hcloud-token" + "webhook-secret" + "admin-token" + "ssh-private-key" + ]); + + # Registration token is shared with the existing K8s/local runner setup. + registrationTokenPath = config.sops.secrets."gitea-runner/org-registration-token".path; + + commonEnvironment = + [ + "GCR_STATE_DIR=/var/lib/gitea-runner-controller" + "GCR_GITEA_URL=${cfg.giteaBaseUrl}" + "GCR_ALLOWED_REPOS=${lib.concatStringsSep "," cfg.allowedRepos}" + "GCR_CONCURRENCY_CAP=${toString cfg.concurrencyCap}" + "GCR_PER_REPO_CAP=${toString cfg.perRepoCap}" + "GCR_RECONCILE_INTERVAL_SEC=${toString cfg.reconcileIntervalSec}" + "GCR_BUDGET_EUR_MONTHLY=${cfg.budgetEurMonthly}" + "GCR_HETZNER_LOCATION=${cfg.hetznerLocation}" + "HCLOUD_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/hcloud-token".path}" + "GITEA_WEBHOOK_SECRET_FILE=${config.sops.secrets."${secretPrefix}/webhook-secret".path}" + "GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}" + "GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}" + "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}" + "GCR_NIX_VERSION=${cfg.nixVersion}" + "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}" + "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}" + "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}" + ] + ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ]; +in +{ + options = { + hectic.services.gitea-runner-controller = { + enable = lib.mkEnableOption "gitea-runner-controller — ephemeral Hetzner VM runner controller"; + listenAddr = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = "Address the webhook listener binds to."; + }; + listenPort = lib.mkOption { + type = lib.types.port; + default = 8787; + description = "Port the webhook listener binds to."; + }; + webhookHost = lib.mkOption { + type = lib.types.str; + default = "runners.hectic-lab.com"; + description = "Public vhost Gitea delivers webhooks to. Requires a DNS A record to this host."; + }; + giteaBaseUrl = lib.mkOption { + type = lib.types.str; + default = "https://gitea.hectic-lab.com"; + description = "Public Gitea URL runners register against."; + }; + allowedRepos = lib.mkOption { + type = with lib.types; listOf str; + default = [ "hectic-lab/util.nix" ]; + description = "Repos whose workflow_job events may trigger VM creation."; + }; + concurrencyCap = lib.mkOption { + type = lib.types.int; + default = 2; + description = "Maximum simultaneously running ephemeral VMs (global)."; + }; + perRepoCap = lib.mkOption { + type = lib.types.int; + default = 1; + description = "Maximum concurrent ephemeral VMs per repo."; + }; + reconcileIntervalSec = lib.mkOption { + type = lib.types.int; + default = 60; + description = "Seconds between reconciliation ticks."; + }; + budgetEurMonthly = lib.mkOption { + type = lib.types.str; + default = "15"; + description = "Soft monthly EUR ceiling for estimated VM spend."; + }; + hetznerLocation = lib.mkOption { + type = lib.types.str; + default = "nbg1"; + description = "Hetzner location for ephemeral VMs."; + }; + imageId = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "174108912"; + description = '' + Hetzner image/snapshot id for ephemeral VMs (MicroOS base from + gitea-runners-build-microos-snapshots). Controller refuses VM + creation while null. + ''; + }; + actRunnerVersion = lib.mkOption { + type = lib.types.str; + default = "v0.2.11"; + description = "act_runner release tag downloaded at VM bootstrap."; + }; + actRunnerSha256 = lib.mkOption { + type = lib.types.str; + default = "7a5e833793286bbfd9b59ce682bd41fc3f1c096bae1bb2a09b66ab2f6dacf90c"; + description = "sha256 of the pinned act_runner linux-amd64 binary, verified at bootstrap."; + }; + nixVersion = lib.mkOption { + type = lib.types.str; + default = "2.28.3"; + description = "Nix release installed from the official static tarball at bootstrap."; + }; + nixTarballSha256 = lib.mkOption { + type = lib.types.str; + default = "85d1847d06d5d56167796d3f61cd992908de84584db3e700da031a782b59ea22"; + description = "sha256 of the pinned Nix x86_64-linux tarball, verified at bootstrap."; + }; + debugSshPublicKey = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + Injected into every ephemeral VM (project ssh-key "yukkop@nixos" + carries the matching public key; this value is informational and + used by gcr_bootstrap_script documentation). + ''; + }; + bootstrapSshPrivateKeyFile = lib.mkOption { + type = lib.types.path; + description = '' + Path (sops-rendered) to the SSH private key the controller uses to + push bootstrap into ephemeral VMs. Public half must be registered + as Hetzner project ssh-key "yukkop@nixos". + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.imageId != null; + message = "gitea-runner-controller: imageId must be set to a MicroOS snapshot id before enabling"; + } + ]; + + sops.secrets = controllerSecrets; + + systemd.services.gitea-runner-controller = { + description = "Gitea ephemeral runner controller — reconcile loop"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "simple"; + ExecStart = "${self.packages.${system}.gitea-runner-controller}/bin/gitea-runner-controller"; + Restart = "always"; + RestartSec = "5s"; + StateDirectory = "gitea-runner-controller"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + StandardOutput = "journal"; + StandardError = "journal"; + Environment = commonEnvironment; + }; + }; + + systemd.services.gitea-runner-webhook = { + description = "Gitea ephemeral runner webhook receiver"; + after = [ + "network.target" + "gitea-runner-controller.service" + ]; + wantedBy = [ "multi-user.target" ]; + partOf = [ "gitea-runner-controller.service" ]; + serviceConfig = { + Type = "simple"; + ExecStart = "${self.packages.${system}.gitea-runner-controller}/bin/gitea-runner-webhook"; + Restart = "always"; + RestartSec = "2s"; + StateDirectory = "gitea-runner-controller"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + StandardOutput = "journal"; + StandardError = "journal"; + Environment = commonEnvironment ++ [ + "GCR_LISTEN_ADDR=${cfg.listenAddr}" + "GCR_LISTEN_PORT=${toString cfg.listenPort}" + ]; + }; + }; + + services.nginx.virtualHosts."${cfg.webhookHost}" = { + enableACME = true; + forceSSL = true; + locations."/" = { + extraConfig = '' + proxy_pass http://${cfg.listenAddr}:${toString cfg.listenPort}; + proxy_read_timeout 30s; + ''; + }; + }; + }; +} diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index 301ac560..ab1ea3c8 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -92,6 +92,19 @@ in { services.matrix = { enable = false; }; + services.gitea-runner-controller = { + # NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job). + # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover". + enable = true; + imageId = "424558114"; # MicroOS x86 + Hetzner datasource dhcpcd fix + allowedRepos = [ + "hectic-lab/util.nix" + "hectic-lab/runner-clean" + "hectic-lab/runner-clean2" + ]; + # FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable. + debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ yukkop@nixos"; + }; }; # NOTE(yukkop): disk was provisioned by Hetzner rescue image, disko was never @@ -294,6 +307,8 @@ in { virtualHosts."gitea.${domain}" = { enableACME = true; forceSSL = true; + # NOTE(yukkop): allow large git pushes over HTTPS. + extraConfig = "client_max_body_size 512m;"; locations."/" = { extraConfig = '' proxy_pass http://127.0.0.1:11011/; @@ -323,7 +338,7 @@ in { }; }; gitea-actions-runner.instances.${giteaRunnerInstance} = { - enable = true; + enable = false; name = giteaRunnerInstance; url = "https://gitea.${domain}"; tokenFile = giteaRunnerTokenEnv; diff --git a/package/default.nix b/package/default.nix index 9b6d23ef..5c7ba5e0 100644 --- a/package/default.nix +++ b/package/default.nix @@ -144,6 +144,7 @@ in { support-bot = pkgs.callPackage ./support-bot {}; gitea-heatmap = pkgs.callPackage ./gitea {}; gitea-runner-nix-image = pkgs.callPackage ./gitea-runner-nix-image {}; + gitea-runner-controller = pkgs.callPackage ./gitea-runner-controller {}; nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {}; "sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {}; deploy = pkgs.callPackage ./deploy { inherit inputs; }; diff --git a/package/gitea-runner-controller/controller.sh b/package/gitea-runner-controller/controller.sh new file mode 100644 index 00000000..db1b71a4 --- /dev/null +++ b/package/gitea-runner-controller/controller.sh @@ -0,0 +1,219 @@ +#!/bin/dash +# Reconcile loop for gitea-runner-controller. +# Owns: TTL sweep, orphan-VM sweep, deferred-job retry, stale-runner dereg, +# startup convergence. Runs forever under systemd; webhook service is separate. + +gcr_ttl_grace_sec() { + printf '%s' "$((10 * 60))" +} + +gcr_record_age_sec() { + created_at="$(gcr_record_field "$1" created_at)" + now="$(date -u '+%s')" + case "$created_at" in + ''|*[!0-9]*) printf '%s' 999999 ;; + *) printf '%s' "$((now - created_at))" ;; + esac +} + +gcr_sweep_ttl() { + for f in $(gcr_active_records); do + rec="$(cat "$f")" + status="$(gcr_record_field "$rec" status)" + [ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue + + job_id="$(gcr_record_field "$rec" job_id)" + attempt="$(gcr_record_field "$rec" run_attempt)" + ttl_min="$(gcr_record_field "$rec" ttl_min)" + case "$ttl_min" in ''|*[!0-9]*) continue ;; esac + + max_sec="$((ttl_min * 60 + $(gcr_ttl_grace_sec)))" + age="$(gcr_record_age_sec "$rec")" + if [ "$age" -gt "$max_sec" ]; then + vm_id="$(gcr_record_field "$rec" vm_id)" + gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s" + if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then + gcr_vm_destroy "$vm_id" || true + gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}" + fi + gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}" + gcr_record_del "$job_id" "$attempt" + gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")" + fi + done +} + +gcr_sweep_orphan_vms() { + vms_json="$(gcr_vm_list_managed)" || return 0 + count="$(printf '%s' "$vms_json" | jq 'length')" + i=0 + while [ "$i" -lt "$count" ]; do + vm="$(printf '%s' "$vms_json" | jq -c ".[$i]")" + vm_id="$(printf '%s' "$vm" | jq -r '.id')" + jid="$(printf '%s' "$vm" | jq -r '.labels["gcr.job-id"] // ""')" + att="$(printf '%s' "$vm" | jq -r '.labels["gcr.run-attempt"] // ""')" + + known="" + if [ -n "$jid" ] && [ -n "$att" ]; then + rec="$(gcr_record_get "$jid" "$att")" + [ -n "$rec" ] && known=1 + fi + + if [ -z "$known" ]; then + gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy" + gcr_vm_destroy "$vm_id" || true + gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}" + fi + i=$((i + 1)) + done +} + +gcr_alloc_deferred() { + job_id="$1"; attempt="$2" + + rec="$(gcr_record_get "$job_id" "$attempt")" + [ -n "$rec" ] || return 0 + [ "$(gcr_record_field "$rec" status)" = "deferred" ] || return 0 + + repo="$(gcr_record_field "$rec" repo)" + label="$(gcr_record_field "$rec" label)" + + profile="$(gcr_label_profile "$label")" || return 0 + set -- $profile + server_type="$1"; ttl_min="$2"; rate="$3" + + active="$(gcr_count_active)" + repo_active="$(gcr_count_active_repo "$repo")" + [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] && return 0 + [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ] && return 0 + + gcr_budget_add "$rate" "$ttl_min" || return 0 + reg_token="$(gcr_gitea_registration_token)" || return 0 + + key="$(gcr_alloc_key "$job_id" "$attempt")" + gcr_lock_acquire "$key" || return 0 + + vm_name="gcr-${job_id}-${attempt}" + vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \ + "$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || { + gcr_lock_release "$key" + return 0 + } + + rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \ + --arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \ + --arg vn "$vm_name" --arg ttl "$ttl_min" \ + '{job_id:$j, run_attempt:$a, repo:$r, label:$l, + created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber), + vm_name:$vn, bootstrapped:false, status:"pending_vm"}')" + gcr_record_put "$job_id" "$attempt" "$rec" + gcr_lock_release "$key" + gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min,\"via\":\"deferred-retry\"}" + gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id" +} + +gcr_retry_deferred() { + for f in $(gcr_active_records); do + rec="$(cat "$f")" + [ "$(gcr_record_field "$rec" status)" = "deferred" ] || continue + gcr_alloc_deferred \ + "$(gcr_record_field "$rec" job_id)" \ + "$(gcr_record_field "$rec" run_attempt)" + done +} + +gcr_sweep_stale_runners() { + runners="$(gcr_gitea_list_runners)" || return 0 + # Here-doc instead of pipe: dash runs pipe tails in a subshell, which + # would strand gcr_event/audit writes from the caller's perspective. + while read -r rid rname; do + [ -n "${rid:-}" ] || continue + case "$rname" in + gcr-*) ;; + *) continue ;; + esac + + # gcr--: alive iff a matching active/pending record exists. + rest="${rname#gcr-}" + jid="${rest%-*}" + att="${rest##*-}" + rec="" + case "$jid" in *[!0-9]*|"") rec="" ;; + *) case "$att" in *[!0-9]*|"") rec="" ;; + *) rec="$(gcr_record_get "$jid" "$att")" ;; + esac ;; + esac + + if [ -z "$rec" ]; then + gcr_log warn --ns=sweep "stale runner registration id=$rid name=$rname -> delete" + if gcr_gitea_delete_runner "$rid"; then + gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"runner_id\":$rid,\"name\":\"$rname\"}" + else + gcr_log error --ns=sweep "failed deleting runner id=$rid" + fi + fi + done < ipv4 or empty + if gcr_hcloud_req GET "/servers/$1"; then + jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY" + fi +} + +# Runs SSH-push bootstrap for VMs that were created but not yet provisioned. +# Registration token is fetched fresh per attempt (short-lived usefulness). +gcr_bootstrap_pending() { + for f in $(gcr_active_records); do + rec="$(cat "$f")" + [ "$(gcr_record_field "$rec" status)" = "pending_vm" ] || continue + [ "$(gcr_record_field "$rec" bootstrapped)" = "true" ] && continue + + job_id="$(gcr_record_field "$rec" job_id)" + attempt="$(gcr_record_field "$rec" run_attempt)" + label="$(gcr_record_field "$rec" label)" + vm_id="$(gcr_record_field "$rec" vm_id)" + runner_name="$(gcr_record_field "$rec" vm_name)" + + ip="$(gcr_vm_public_ip "$vm_id")" + [ -n "$ip" ] || continue + + reg_token="$(gcr_gitea_registration_token)" || continue + ttl_min="$(gcr_record_field "$rec" ttl_min)" + + gcr_log info --ns=alloc "bootstrapping vm=$vm_id ip=$ip job=$job_id" + if gcr_vm_bootstrap_ssh "$ip" "$label" "$reg_token" "$runner_name"; then + rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | .ip = $ip' --arg ip "$ip")" + gcr_record_put "$job_id" "$attempt" "$rec" + gcr_event "vm-bootstrapped" "$job_id" "{\"vm_id\":$vm_id,\"ip\":\"$ip\"}" + else + gcr_log warn --ns=alloc "bootstrap failed vm=$vm_id (retry next tick)" + fi + done +} + +gcr_tick() { + gcr_sweep_ttl + gcr_sweep_orphan_vms + gcr_retry_deferred + gcr_bootstrap_pending + gcr_sweep_stale_runners +} + +gcr_main() { + : "${GCR_RECONCILE_INTERVAL_SEC:=60}" + gcr_state_init + + gcr_log info --ns=core "controller starting, interval=${GCR_RECONCILE_INTERVAL_SEC}s state=$GCR_STATE_DIR" + gcr_tick + + while :; do + sleep "$GCR_RECONCILE_INTERVAL_SEC" + if ! gcr_tick; then + gcr_log error --ns=core "tick failed, retrying next interval" + fi + done +} diff --git a/package/gitea-runner-controller/decide.sh b/package/gitea-runner-controller/decide.sh new file mode 100644 index 00000000..dcc867de --- /dev/null +++ b/package/gitea-runner-controller/decide.sh @@ -0,0 +1,70 @@ +#!/bin/dash +# Allocation decision for gitea-runner-controller. +# Fail-closed: anything not explicitly allowed here is refused. +# +# gcr_decide LABEL REPO -> prints " " and +# returns 0 when allowed; returns 1 with reason on stderr otherwise. + +gcr_label_profile() { + case "$1" in + nix) printf 'cx33 180 0.008' ;; + ubuntu-latest) printf 'cx33 60 0.008' ;; + *) return 1 ;; + esac +} + +gcr_repo_allowed() { + repo="$1" + oldIFS="$IFS" + IFS=, + for allowed in ${GCR_ALLOWED_REPOS:-}; do + if [ "$allowed" = "$repo" ]; then + IFS="$oldIFS" + return 0 + fi + done + IFS="$oldIFS" + return 1 +} + +gcr_decide() { + label="$1"; repo="$2" + + if ! gcr_repo_allowed "$repo"; then + gcr_log warn --ns=decide "repo not allowed: $repo" + return 1 + fi + + # Multi-label jobs are out of MVP scope: ambiguous VM profile mapping. + if ! profile="$(gcr_label_profile "$label")"; then + gcr_log warn --ns=decide "unknown or unsupported label: $label" + return 1 + fi + + printf '%s\n' "$profile" +} + +gcr_count_active() { + count=0 + for f in $(gcr_active_records); do + status="$(gcr_record_field "$(cat "$f")" status)" + case "$status" in + pending_vm|vm_active) count=$((count + 1)) ;; + esac + done + printf '%s' "$count" +} + +gcr_count_active_repo() { + repo="$1" + count=0 + for f in $(gcr_active_records); do + rec="$(cat "$f")" + case "$(gcr_record_field "$rec" status)" in + pending_vm|vm_active) ;; + *) continue ;; + esac + [ "$(gcr_record_field "$rec" repo)" = "$repo" ] && count=$((count + 1)) + done + printf '%s' "$count" +} diff --git a/package/gitea-runner-controller/default.nix b/package/gitea-runner-controller/default.nix new file mode 100644 index 00000000..e8f9f1f3 --- /dev/null +++ b/package/gitea-runner-controller/default.nix @@ -0,0 +1,92 @@ +{ + symlinkJoin, + hectic, + dash, + socat, + curl, + jq, + coreutils, + gawk, + gnugrep, + gnused, + openssl, +}: +let + shell = "${dash}/bin/dash"; + bashOptions = [ + "errexit" + "nounset" + ]; + # SC2329: lib units are sourced wholesale into both binaries; the reconciler + # and webhook each leave a few wrapper functions unreferenced by design. + excludeShellChecks = [ + "SC2086" # word splitting on purpose: env lists and profile triples + "SC2046" # same, command substitution into set -- + "SC2329" + ]; + + runtimeDeps = [ + curl + jq + coreutils + gawk + gnugrep + gnused + openssl + ]; + + lib = '' + ${builtins.readFile ./log.sh} + ${builtins.readFile ./state.sh} + ${builtins.readFile ./decide.sh} + ${builtins.readFile ./hcloud.sh} + ${builtins.readFile ./gitea.sh} + ''; + + handler = hectic.writeShellApplication { + inherit shell bashOptions; + inherit excludeShellChecks; + name = "gcr-webhook-handler"; + runtimeInputs = [ socat ] ++ runtimeDeps; + text = '' + ${lib} + ${builtins.readFile ./webhook.sh} + gcr_state_init + gcr_handle_webhook || gcr_respond 500 "internal error" + exit 0 + ''; + }; + + webhook = hectic.writeShellApplication { + inherit shell bashOptions; + inherit excludeShellChecks; + name = "gitea-runner-webhook"; + runtimeInputs = [ socat ]; + text = '' + : "''${GCR_LISTEN_ADDR:=127.0.0.1}" + : "''${GCR_LISTEN_PORT:=8787}" + exec ${socat}/bin/socat -T5 -t5 \ + "TCP-LISTEN:$GCR_LISTEN_PORT,bind=$GCR_LISTEN_ADDR,reuseaddr,fork" \ + EXEC:"${handler}/bin/gcr-webhook-handler",pipes + ''; + }; + + controller = hectic.writeShellApplication { + inherit shell bashOptions; + inherit excludeShellChecks; + name = "gitea-runner-controller"; + runtimeInputs = runtimeDeps; + text = '' + ${lib} + ${builtins.readFile ./controller.sh} + gcr_main + ''; + }; +in +symlinkJoin { + name = "gitea-runner-controller"; + paths = [ + controller + webhook + ]; +} diff --git a/package/gitea-runner-controller/gitea.sh b/package/gitea-runner-controller/gitea.sh new file mode 100644 index 00000000..af1fd96b --- /dev/null +++ b/package/gitea-runner-controller/gitea.sh @@ -0,0 +1,33 @@ +#!/bin/dash +# Gitea API wrappers for gitea-runner-controller. +# Requires: GCR_GITEA_URL, GITEA_REGISTRATION_TOKEN_FILE, GITEA_ADMIN_TOKEN_FILE + +gcr_gitea_registration_token() { + token="$(gcr_gitea_admin_token)" || return 1 + curl -fsS -X POST -H "Authorization: token $token" \ + "$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners/registration-token" \ + | jq -r '.token' +} + +gcr_gitea_admin_token() { + test -r "${GITEA_ADMIN_TOKEN_FILE:-}" || { + gcr_log error --ns=gitea "GITEA_ADMIN_TOKEN_FILE missing" + return 1 + } + tr -d '\n' < "$GITEA_ADMIN_TOKEN_FILE" +} + +# gcr_gitea_list_runners — prints "id name" lines for org hectic-lab. +gcr_gitea_list_runners() { + token="$(gcr_gitea_admin_token)" || return 1 + curl -fsS -H "Authorization: token $token" \ + "$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners?per_page=50" \ + | jq -r '.entries[] | "\(.id) \(.name)"' +} + +gcr_gitea_delete_runner() { + id="$1" + token="$(gcr_gitea_admin_token)" || return 1 + curl -fsS -X DELETE -H "Authorization: token $token" \ + "$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners/$id" +} diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh new file mode 100644 index 00000000..7719a87a --- /dev/null +++ b/package/gitea-runner-controller/hcloud.sh @@ -0,0 +1,323 @@ +#!/bin/dash +# Hetzner Cloud API wrappers for gitea-runner-controller. +# Requires: HCLOUD_TOKEN_FILE, GCR_HETZNER_LOCATION, GCR_IMAGE_ID, +# GCR_ACT_RUNNER_VERSION, GCR_ACT_RUNNER_SHA256, GCR_NIX_VERSION, +# GCR_NIX_TARBALL_SHA256, GCR_GITEA_URL +# All VMs carry the tag pair gitea-runner-controller=managed plus gcr.* metadata. + +GCR_API="https://api.hetzner.cloud/v1" + +gcr_hcloud_token() { + test -n "${HCLOUD_TOKEN_FILE:-}" && test -r "$HCLOUD_TOKEN_FILE" || { + gcr_log error --ns=hcloud "HCLOUD_TOKEN_FILE missing or unreadable" + return 1 + } + tr -d '\n' < "$HCLOUD_TOKEN_FILE" +} + +# All request state flows through files/exit codes, never command substitution +# ($( ) runs in a subshell and would strand GCR_REQ_FAILED/GCR_LAST_HTTP). +gcr_hcloud_req() { + # gcr_hcloud_req METHOD PATH [JSON_BODY] + # Body written to $GCR_LAST_BODY; exit 0 only on HTTP 2xx. + method="$1"; path="$2"; body="${3:-}" + token="$(gcr_hcloud_token)" || return 1 + GCR_LAST_BODY="$(mktemp "${TMPDIR:-/tmp}/gcr-resp.XXXXXX")" + if [ -n "$body" ]; then + code="$(printf '%s' "$body" | curl -sS -X "$method" \ + -H "Authorization: Bearer $token" \ + -H 'Content-Type: application/json' \ + --data-binary @- \ + -o "$GCR_LAST_BODY" \ + -w '%{http_code}' \ + "$GCR_API$path")" + else + code="$(curl -sS -X "$method" \ + -H "Authorization: Bearer $token" \ + -o "$GCR_LAST_BODY" \ + -w '%{http_code}' \ + "$GCR_API$path")" + fi + case "$code" in 2??) return 0 ;; esac + gcr_log warn --ns=hcloud "request failed path=$path http=$code body=$(head -c 200 "$GCR_LAST_BODY" | gcr_redact)" + return 1 +} + +gcr_vm_list_managed() { + if gcr_hcloud_req GET "/servers?label_selector=gitea-runner-controller%3Dmanaged&per_page=50"; then + jq -S '.servers' "$GCR_LAST_BODY" + fi +} + +gcr_vm_build_userdata() { + vm_name="$1"; label="$2"; reg_token="$3" + + nix_conf='accept-flake-config = true +experimental-features = nix-command flakes +substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA= +sandbox = false' + + runner_config="log: + level: info +runner: + file: /var/lib/gitea-runner/.runner + capacity: 1 + timeout: $(printf '%s' "$(gcr_label_profile "$label")" | awk '{print $2}')m + insecure: false + fetch_timeout: 5s + fetch_interval: 2s +labels: + - \"$label:host\"" + + ssh_key_block="" + if [ -n "${GCR_DEBUG_SSH_PUBKEY:-}" ]; then + ssh_key_block=" - path: /root/.ssh/authorized_keys + permissions: '0600' + content: | + $GCR_DEBUG_SSH_PUBKEY" + fi + + # NOTE(yukkop): token reaches only this VM's Hetzner metadata service; + # ephemeral registration makes it useless after the single job exits. + printf '%s' "#cloud-config +write_files: +$ssh_key_block + - path: /etc/ssh/sshd_config.d/99-gcr-root.conf + permissions: '0644' + content: | + PermitRootLogin prohibit-password + PubkeyAuthentication yes + - path: /etc/nix/nix.conf + content: | +$(printf '%s\n' "$nix_conf" | sed 's/^/ /') + - path: /etc/gitea-runner/config.yaml + content: | +$(printf '%s\n' "$runner_config" | sed 's/^/ /') + - path: /etc/systemd/system/gitea-runner.service + content: | + [Unit] + Description=Gitea ephemeral Actions runner + After=network-online.target gcr-bootstrap.service + Requires=gcr-bootstrap.service + + [Service] + Type=simple + Environment=GITEA_INSTANCE_URL=$GCR_GITEA_URL + Environment=GITEA_RUNNER_REGISTRATION_TOKEN=$reg_token + ExecStart=/usr/local/bin/act_runner daemon --ephemeral --config /etc/gitea-runner/config.yaml + Restart=on-failure + RestartSec=5 + + [Install] + WantedBy=multi-user.target + - path: /etc/systemd/system/gcr-bootstrap.service + content: | + [Unit] + Description=Bootstrap Nix + act_runner for ephemeral CI job + After=network-online.target + Wants=network-online.target + Before=gitea-runner.service + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/usr/local/sbin/gcr-bootstrap + + [Install] + WantedBy=multi-user.target + - path: /usr/local/sbin/gcr-bootstrap + permissions: '0700' + content: | + #!/bin/sh + set -eu + exec > /var/log/gcr-bootstrap.log 2>&1 + curl -fsSL \"https://nixos.org/releases/nix/$GCR_NIX_VERSION/nix-$GCR_NIX_VERSION-x86_64-linux.tar.xz\" -o /tmp/nix.tar.xz + printf '%s /tmp/nix.tar.xz\n' \"$GCR_NIX_TARBALL_SHA256\" | sha256sum -c - + tar -xJf /tmp/nix.tar.xz -C /tmp + /tmp/nix-$GCR_NIX_VERSION-x86_64-linux/install --no-daemon + rm -rf /tmp/nix* + curl -fsSL \"https://gitea.com/gitea/act_runner/releases/download/$GCR_ACT_RUNNER_VERSION/act_runner-\$(printf '%s' \"$GCR_ACT_RUNNER_VERSION\" | sed 's/^v//')-linux-amd64\" -o /usr/local/bin/act_runner + printf '%s /usr/local/bin/act_runner\n' \"$GCR_ACT_RUNNER_SHA256\" | sha256sum -c - + chmod 0755 /usr/local/bin/act_runner + mkdir -p /var/lib/gitea-runner +runcmd: + - [ sh, -c, 'systemctl enable --now sshd.service 2>/dev/null || systemctl enable --now ssh 2>/dev/null || true' ] + - [ sh, -c, 'systemctl restart sshd.service 2>/dev/null || systemctl restart ssh 2>/dev/null || true' ] + - [ systemctl, enable, --now, gitea-runner.service ] +" +} + +# gcr_vm_create NAME LABEL SERVER_TYPE TTL_MIN REG_TOKEN JOB_ID ATTEMPT REPO +# Prints new server id. +gcr_vm_create() { + vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4" + reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8" + + test -n "${GCR_IMAGE_ID:-}" || { + gcr_log error --ns=hcloud "GCR_IMAGE_ID not set; refusing VM creation" + return 1 + } + + userdata="$(gcr_vm_build_userdata "$vm_name" "$label" "$reg_token")" + payload="$(jq -n \ + --arg name "$vm_name" \ + --arg stype "$server_type" \ + --arg image "$GCR_IMAGE_ID" \ + --arg loc "${GCR_HETZNER_LOCATION:-nbg1}" \ + --arg udata "$userdata" \ + --arg jid "$job_id" \ + --arg att "$attempt" \ + --arg repo "$repo" \ + --arg label "$label" \ + --arg ts "$(date -u '+%s')" \ + --arg ttl "$ttl_min" \ + --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \ + '{name:$name, server_type:$stype, image:$image, location:$loc, + start_after_create:true, + labels:{ + "gitea-runner-controller":"managed", + "gcr.job-id":$jid, "gcr.run-attempt":$att, + "gcr.repo":$repo_safe, "gcr.label":$label, + "gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')" + + # Hetzner placement is occasionally transient (resource_unavailable); + # retry a few times before giving up. NOTE: userdata/cloud-init is NOT + # used — bootstrap happens over SSH from the controller (see + # gcr_vm_bootstrap_ssh); MicroOS snapshot's Hetzner datasource cannot + # fetch user-data (DHCP Exception on this image lineage). + attempt_n=0 + while :; do + attempt_n=$((attempt_n + 1)) + if gcr_hcloud_req POST /servers "$payload"; then + jq -r '.server.id' "$GCR_LAST_BODY" + return 0 + fi + gcr_log warn --ns=hcloud "create attempt=$attempt_n failed" + [ "$attempt_n" -ge 3 ] && return 1 + sleep $((attempt_n * 10)) + done +} + +# gcr_vm_destroy SERVER_ID — idempotent best-effort destroy. +gcr_vm_destroy() { + if ! gcr_hcloud_req DELETE "/servers/$1"; then + gcr_log warn --ns=hcloud "destroy failed or already gone: server $1" + return 1 + fi +} + +# Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's +# cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the +# controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose +# public half is authorized on every ephemeral VM (project ssh-key injection). +gcr_bootstrap_script() { + # gcr_bootstrap_script LABEL REG_TOKEN TTL_MIN RUNNER_NAME -> POSIX sh payload + label="$1"; reg_token="$2"; ttl_min="$3"; runner_name="$4" + nix_conf='accept-flake-config = true +experimental-features = nix-command flakes +substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA= +sandbox = false' + + runner_config="log: + level: info +runner: + file: /var/lib/gitea-runner/.runner + capacity: 1 + timeout: ${ttl_min}m + insecure: false + fetch_timeout: 5s + fetch_interval: 2s +labels: + - \"$label:host\"" + + cat </var/log/gcr-bootstrap.log 2>&1 +set -eu +mkdir -p /etc/nix /etc/gitea-runner /var/lib/gitea-runner /usr/local/bin +cat > /etc/nix/nix.conf <<'NIXEOF' +$nix_conf +NIXEOF +cat > /etc/gitea-runner/config.yaml <<'CFGEOF' +$runner_config +CFGEOF +cat > /usr/local/sbin/gcr-runner-start < /etc/systemd/system/gitea-runner.service < /usr/local/sbin/gcr-install < "$key_tmp" + printf '\n' >> "$key_tmp" + chmod 0600 "$key_tmp" + SSH_OPTS="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes" + + waited=0 + until ssh $SSH_OPTS "root@$ip" true 2>/dev/null; do + waited=$((waited + 5)) + [ "$waited" -ge 300 ] && { + gcr_log warn --ns=hcloud "sshd never came up on $ip" + rm -f "$key_tmp" + return 1 + } + sleep 5 + done + + ttl_min="$(printf '%s' "$(gcr_label_profile "$label")" | awk '{print $2}')" + script="$(gcr_bootstrap_script "$label" "$reg_token" "$ttl_min" "$runner_name")" + if printf '%s' "$script" | ssh $SSH_OPTS "root@$ip" sh -s; then + rm -f "$key_tmp" + return 0 + fi + rm -f "$key_tmp" + return 1 +} diff --git a/package/gitea-runner-controller/log.sh b/package/gitea-runner-controller/log.sh new file mode 100644 index 00000000..6b73d38a --- /dev/null +++ b/package/gitea-runner-controller/log.sh @@ -0,0 +1,77 @@ +#!/bin/dash +# Log helper for gitea-runner-controller. +# Verbosity via GCR_LOG env: "[;=]..." e.g. "info;alloc=debug". + +MAGENTA='\033[0;35m' +BLUE='\033[0;34m' +GREEN='\033[0;32m' +CYAN='\033[0;36m' +YELLOW='\033[0;33m' +RED='\033[0;31m' +WHITE='\033[0;37m' +NC='\033[0m' + +gcr_log_enabled() { + level="$1" + ns="${2:-core}" + conf="${GCR_LOG:-info}" + ns_level="" + default_level="" + oldIFS="$IFS" + IFS=';' + for pair in $conf; do + case "$pair" in + *=*) ns_name="${pair%%=*}"; ns_level="${pair#*=}" + [ "$ns_name" = "$ns" ] && { IFS="$oldIFS"; echo "$ns_level"; return 0; } ;; + *) default_level="$pair" ;; + esac + done + IFS="$oldIFS" + echo "${default_level:-info}" +} + +gcr_log_level_rank() { + case "$1" in + trace) echo 0 ;; + debug) echo 1 ;; + info) echo 2 ;; + notice) echo 3 ;; + warn) echo 4 ;; + error) echo 5 ;; + panic) echo 6 ;; + *) echo 7 ;; + esac +} + +gcr_log() { + level="$1"; shift + ns="core" + case "$1" in + --ns=*) ns="${1#--ns=}"; shift ;; + esac + + want="$(gcr_log_enabled "$level" "$ns")" + [ "$(gcr_log_level_rank "$level")" -ge "$(gcr_log_level_rank "$want")" ] || return 0 + + ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + msg="$1" + case "$level" in + trace) color="$WHITE" ;; + debug) color="$BLUE" ;; + info) color="$GREEN" ;; + notice) color="$CYAN" ;; + warn) color="$YELLOW" ;; + error) color="$RED" ;; + panic) color="$MAGENTA" ;; + *) color="$WHITE" ;; + esac + printf '%b\n' "${color}${ts} ${level}[${ns}]${NC} $msg" >&2 +} + +# Security: keep credential values out of logs; keys containing TOKEN/SECRET +# and Authorization token headers are rewritten to . +gcr_redact() { + sed -E \ + -e 's/([A-Za-z0-9_]*(TOKEN|SECRET)[A-Za-z0-9_]*)(=|: ?"?)[^ "]+/\1\3/g' \ + -e 's/(Authorization: *token )[^ ]+/\1/g' +} diff --git a/package/gitea-runner-controller/state.sh b/package/gitea-runner-controller/state.sh new file mode 100644 index 00000000..aa447560 --- /dev/null +++ b/package/gitea-runner-controller/state.sh @@ -0,0 +1,74 @@ +#!/bin/dash +# State primitives for gitea-runner-controller. +# Layout: +# $GCR_STATE_DIR/jobs/.json allocation records +# $GCR_STATE_DIR/jobs/.lock./ mkdir(2) atomicity guards +# $GCR_STATE_DIR/events.jsonl append-only audit +# $GCR_STATE_DIR/budget/ estimated EUR spent this month + +gcr_state_init() { + test -n "${GCR_STATE_DIR:-}" || { echo "GCR_STATE_DIR is not set" >&2; return 1; } + mkdir -p "$GCR_STATE_DIR/jobs" "$GCR_STATE_DIR/budget" + touch "$GCR_STATE_DIR/events.jsonl" +} + +gcr_alloc_key() { + printf '%s-%s' "$1" "$2" +} + +gcr_record_path() { + printf '%s/jobs/%s.json' "$GCR_STATE_DIR" "$(gcr_alloc_key "$1" "$2")" +} + +# mkdir(2) atomicity guard: succeeds exactly once per key until released. +gcr_lock_acquire() { + mkdir "$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$1")" 2>/dev/null +} + +gcr_lock_release() { + rm -rf "$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$1")" +} + +gcr_record_get() { + # Missing record is a normal answer, not an error; must not trip errexit. + { cat "$(gcr_record_path "$1" "$2")" 2>/dev/null || true; } +} + +# mktemp+mv keeps concurrent readers away from partially written records. +gcr_record_put() { + tmp="$(mktemp "$(dirname "$(gcr_record_path "$1" "$2")")/.tmp.XXXXXX")" + printf '%s\n' "$3" > "$tmp" + mv -f "$tmp" "$(gcr_record_path "$1" "$2")" +} + +gcr_record_del() { + rm -f "$(gcr_record_path "$1" "$2")" +} + +gcr_record_field() { + printf '%s' "$1" | jq -r --arg f "$2" '.[$f] // ""' +} + +gcr_event() { + printf '{"ts":"%s","event":"%s","job_id":"%s","detail":%s}\n' \ + "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$1" "$2" \ + "$(printf '%s' "$3" | jq -Rs .)" >> "$GCR_STATE_DIR/events.jsonl" +} + +# Exit-code contract: 0 = recorded under budget, 1 = would exceed cap. +gcr_budget_add() { + rate="$1"; ttl_min="$2" + month="$(date -u '+%Y-%m')" + file="$GCR_STATE_DIR/budget/$month" + current="$(cat "$file" 2>/dev/null || echo 0)" + projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')" + if awk -v p="$projected" -v b="${GCR_BUDGET_EUR_MONTHLY:-15}" 'BEGIN {exit !(p > b)}'; then + return 1 + fi + printf '%s\n' "$projected" > "$file" + return 0 +} + +gcr_active_records() { + grep -l '"status":"\(pending_vm\|vm_active\|deferred\)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true +} diff --git a/package/gitea-runner-controller/webhook.sh b/package/gitea-runner-controller/webhook.sh new file mode 100644 index 00000000..aaf86062 --- /dev/null +++ b/package/gitea-runner-controller/webhook.sh @@ -0,0 +1,233 @@ +#!/bin/dash +# One-shot HTTP handler for Gitea workflow_job webhooks. +# Invoked per connection by socat; request on stdin, response on stdout. + +RESPONSE_CODE=204 +RESPONSE_BODY="" + +gcr_respond() { + code="$1"; body="${2:-}" + reason="" + case "$code" in + 200) reason="OK" ;; + 202) reason="Accepted" ;; + 204) reason="No Content" ;; + 400) reason="Bad Request" ;; + 403) reason="Forbidden" ;; + 413) reason="Payload Too Large" ;; + *) reason="No Content" ;; + esac + printf 'HTTP/1.1 %s %s\r\n' "$code" "$reason" + printf 'Content-Type: text/plain\r\n' + printf 'Content-Length: %s\r\n' "$(printf '%s' "$body" | wc -c)" + printf 'Connection: close\r\n\r\n' + [ -n "$body" ] && printf '%s\n' "$body" +} + +gcr_read_request() { + request_line="" + gcr_hdr_event_type="" + gcr_hdr_delivery="" + gcr_hdr_signature="" + content_length=0 + + # Correctness depends on dash reading stdin byte-by-byte (no lookahead + # buffer); body bytes must remain unconsumed for `head -c` below. + IFS= read -r request_line || return 1 + + while :; do + IFS= read -r line || break + line="$(printf '%s' "$line" | tr -d '\r')" + [ -z "$line" ] && break + name="$(printf '%s' "$line" | cut -d: -f1 | tr '[:upper:]' '[:lower:]')" + value="$(printf '%s' "${line#*:}" | sed 's/^ *//')" + case "$name" in + x-gitea-event-type) gcr_hdr_event_type="$value" ;; + x-gitea-delivery) gcr_hdr_delivery="$value" ;; + x-gitea-signature) gcr_hdr_signature="$value" ;; + content-length) content_length="$value" ;; + esac + done + + case "$request_line" in + "POST "*" HTTP/"*) ;; + *) return 1 ;; + esac + + case "$content_length" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$content_length" -le 65536 ] || { gcr_respond 413 "payload too large"; exit 0; } + + gcr_body="$(head -c "$content_length")" +} + +gcr_verify_signature() { + test -r "${GITEA_WEBHOOK_SECRET_FILE:-}" || { + gcr_log error --ns=webhook "GITEA_WEBHOOK_SECRET_FILE missing" + return 1 + } + secret="$(cat "$GITEA_WEBHOOK_SECRET_FILE")" + expected="$(printf '%s' "$gcr_body" \ + | openssl dgst -sha256 -hmac "$secret" -hex \ + | awk '{print $NF}')" + # NOTE(yukkop): shell string compare is not constant-time; acceptable here + # because the secret is high-entropy and bodies are signed, not encrypted. + [ "$expected" = "$gcr_hdr_signature" ] +} + +gcr_alloc() { + job_id="$1"; attempt="$2"; repo="$3"; labels_json="$4" + + label="$(printf '%s' "$labels_json" | jq -r '.[0] // ""')" + label_count="$(printf '%s' "$labels_json" | jq 'length')" + + existing="$(gcr_record_get "$job_id" "$attempt")" + if [ -n "$existing" ]; then + gcr_log debug --ns=alloc "duplicate delivery for $(gcr_alloc_key "$job_id" "$attempt")" + RESPONSE_CODE=204 + return 0 + fi + + key="$(gcr_alloc_key "$job_id" "$attempt")" + if ! gcr_lock_acquire "$key"; then + RESPONSE_CODE=204 + return 0 + fi + + if [ "$label_count" -ne 1 ]; then + gcr_lock_release "$key" + gcr_event "refused" "$job_id" "{\"repo\":\"$repo\",\"label_count\":$label_count}" + RESPONSE_CODE=202; RESPONSE_BODY="refused: exactly one label required" + return 0 + fi + + if ! profile="$(gcr_decide "$label" "$repo")"; then + gcr_lock_release "$key" + gcr_event "refused" "$job_id" "{\"repo\":\"$repo\",\"label\":\"$label\"}" + RESPONSE_CODE=202; RESPONSE_BODY="refused: repo or label not allowed" + return 0 + fi + + set -- $profile + server_type="$1"; ttl_min="$2"; rate="$3" + + active="$(gcr_count_active)" + repo_active="$(gcr_count_active_repo "$repo")" + if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \ + || [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then + rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \ + --arg l "$label" --arg t "$(date -u '+%s')" \ + '{job_id:$j, run_attempt:$a, repo:$r, label:$l, + created_at:$t, ttl_min:null, vm_id:"", vm_name:"", + status:"deferred"}')" + gcr_record_put "$job_id" "$attempt" "$rec" + gcr_lock_release "$key" + gcr_event "deferred" "$job_id" "{\"active\":$active,\"repo_active\":$repo_active}" + RESPONSE_CODE=202; RESPONSE_BODY="deferred: capacity" + return 0 + fi + + if ! gcr_budget_add "$rate" "$ttl_min"; then + gcr_record_del "$job_id" "$attempt" + gcr_lock_release "$key" + gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}" + RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted" + return 0 + fi + + reg_token="$(gcr_gitea_registration_token)" || { + gcr_record_del "$job_id" "$attempt" + gcr_lock_release "$key" + gcr_event "token-error" "$job_id" "{}" + RESPONSE_CODE=202; RESPONSE_BODY="registration token unavailable" + return 0 + } + + vm_name="gcr-${job_id}-${attempt}" + vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \ + "$reg_token" "$job_id" "$attempt" "$repo")" || { + gcr_record_del "$job_id" "$attempt" + gcr_lock_release "$key" + gcr_event "vm-create-failed" "$job_id" "{}" + RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed" + return 0 + } + + rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \ + --arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \ + --arg vn "$vm_name" --arg ttl "$ttl_min" \ + '{job_id:$j, run_attempt:$a, repo:$r, label:$l, + created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber), + vm_name:$vn, bootstrapped:false, status:"pending_vm"}')" + gcr_record_put "$job_id" "$attempt" "$rec" + gcr_lock_release "$key" + gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min}" + + RESPONSE_CODE=202; RESPONSE_BODY="allocated $vm_name" +} + +gcr_deallocate() { + job_id="$1"; attempt="$2"; new_status="$3" + + rec="$(gcr_record_get "$job_id" "$attempt")" + [ -n "$rec" ] || return 0 + + vm_id="$(gcr_record_field "$rec" vm_id)" + if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then + gcr_vm_destroy "$vm_id" || true + gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" + fi + + gcr_record_del "$job_id" "$attempt" + gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")" +} + +gcr_handle_webhook() { + gcr_read_request || { gcr_respond 400 ""; exit 0; } + + [ "$gcr_hdr_event_type" = "workflow_job" ] || { + gcr_log debug --ns=webhook "ignored event type: $gcr_hdr_event_type" + gcr_respond 204 ""; exit 0 + } + + gcr_verify_signature || { + gcr_log warn --ns=webhook "invalid signature, delivery=$gcr_hdr_delivery" + gcr_respond 403 "invalid signature"; exit 0 + } + + action="$(printf '%s' "$gcr_body" | jq -r '.action // ""')" + job_id="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.id // ""')" + attempt="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.run_attempt // ""')" + repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')" + labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')" + + case "$action:$job_id" in + :*|"queued:"|*":0") gcr_respond 400 "malformed payload"; exit 0 ;; + esac + + case "$action" in + queued) + gcr_alloc "$job_id" "$attempt" "$repo" "$labels_json" + gcr_log info --ns=alloc "queued job=$job_id repo=$repo code=$RESPONSE_CODE $RESPONSE_BODY" + ;; + in_progress) + rec="$(gcr_record_get "$job_id" "$attempt")" + if [ -n "$rec" ]; then + rec="$(printf '%s' "$rec" | jq -c '.status = "vm_active"')" + gcr_record_put "$job_id" "$attempt" "$rec" + fi + RESPONSE_CODE=204 + ;; + completed) + gcr_deallocate "$job_id" "$attempt" "completed" + RESPONSE_CODE=204 + ;; + *) + gcr_log debug --ns=webhook "unhandled action: $action" + RESPONSE_CODE=204 + ;; + esac + + gcr_respond "$RESPONSE_CODE" "$RESPONSE_BODY" +} diff --git a/sus/gitea-runners.yaml b/sus/gitea-runners.yaml index d7e8b547..df4cfa85 100644 --- a/sus/gitea-runners.yaml +++ b/sus/gitea-runners.yaml @@ -1,6 +1,11 @@ gitea: hectic-lab: org-runner-registration-token: ENC[AES256_GCM,data:Hez1q44P/gxRfpA5Nl1bsf5Be719z/DadWyJMoghaeh+7dX3TqhFmw==,iv:ufme+uMcsI8pw5DcAlxJJKFbcj7XSg2E4MfC12U3sO0=,tag:JLyyBb5PTLY02jOGeB7EEg==,type:str] + controller: + hcloud-token: ENC[AES256_GCM,data:mw2P6v1k1MOtIpuARG7++T3ica8gY7HdfGOlzr9DyMzqDl7hGAYmpLitVDu++5mZGHZoVboH9YzVwU9d6frGhg==,iv:db2Dy8fZ3vaOlvT5QAxptBY/h2glIroPQq2gpQVIno0=,tag:haR1YKTegFQytkQ6ygAKgw==,type:str] + webhook-secret: ENC[AES256_GCM,data:NppDTo0T+DDlCaNp6mcQWgGfaygFGwDR5bd6oylxFIk/XhSz6ATUpNWSrn9oZm2geuc/BHgGAwcgQU1LnhnJGw==,iv:zhwOxR6Pf/NKSiDME/ipNBhcDize/t4WcSob13QmjsA=,tag:FXJflbfmpgqKZuiEYwBcrg==,type:str] + admin-token: ENC[AES256_GCM,data:inIN+ISgS+HgCEu/r9lfjRbbbCz83NCcwoSaeeqJys6JocEGBy0law==,iv:Lj6f3flrQWatHHy9diMQQmFsnSU0ckqoZB0AX6eL00U=,tag:OazXKYMPbYsy+GE2eVCaNQ==,type:str] + ssh-private-key: ENC[AES256_GCM,data:+RH7sfrKpFo+lVgTunc+qWrqKgbwWHxRDt5iXySLQCeNQR/Qr/S9NWnJgDA4iwZqXxeDq/WlMpKMLfPu7lGeVuMr0sp3D9BCljtRWeFp3/OsWeOaDjHjIxiQF+E6XeaU52lwDaBb3xTUV+4iWZVYHytEtaKkSplxDkNqa+AOxlJ6q5xK1KALJBajR6AhTiLt/IrSDrBCP2Sd79wiBaulcD+WOem25mm8SzbSbnVZ1NnSjvh5O6pKKmRho9yjPVj9yXPad3Hpder986JZSH/wte9koNAaBzEjtAPZd4Do00xboV0PMgFm2aTft2nbDbnpxDMN0wNbcpOmOmGzraxRRTFs8o/UHRsDnWw8ZZGTiqqivCt9q0cB5mVJSxaQpJlzqCX3hjPz8sbCdw/vzIQc76fQHqHcleeb1oht+2E6T2OdRStEDI2u1ts+yF0d2gk/99wYaMK4Oun7qCyffHKsFt/vb1tAT3IXNW0zgkYWJhBd3oUsbV16ImL82VZSDyjIwBFwORkOU6PIUWuqVIo=,iv:k4dQC5FTc1k01z33gVXPTna63l+Ixr1aW7/OXGt7R/A=,tag:/dRO7wRjyYTR3p+6H17w+Q==,type:str] hetzner: hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str] s3: @@ -53,7 +58,7 @@ sops: UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-24T17:35:02Z" - mac: ENC[AES256_GCM,data:Nulehy1TdIx4bGCRLWR6JYKqiyadelKG1fuknfPXArjUGzr4q6Pa+SgvBaKL8zcIlu224SxHNUSo3EfPj/O+f5xPH95hhD1vpbZbBBNsls5gMeVMhSELyfhJUn6e7zSwRGzMrY8wZ4VkBRsitHmMTC+Qg33YXsLEm79GKIhzcKI=,iv:zAhpjVhuMsFN/LHmAUGP2/FpsSlBs++bmkktLBxXQnI=,tag:B0wp4ZWfZ6/+ZyBuaKAQHw==,type:str] + lastmodified: "2026-08-27T07:26:16Z" + mac: ENC[AES256_GCM,data:H71K/HjlVCdaOWLcOsdHJo591aCr5+Fo5ObX2H7yNJAOaN5sE1HH/blz9koxivFEyr/yCr6a9cMZzzQoiZ1M4iFLwjiH/ZkLm44tZJsv9Dc+IJskNO+4Am+31OEgMQaf8ciVSJqn5PRZqPV0bg+OxznShxY/ClxCeIxNINGsvnw=,iv:+FK69Dm5XnPsu4CX+MFILA1Ag+SGJNBV0g1TiT7snRI=,tag:5OX0VVTipocpfjZZ+Q+pLA==,type:str] unencrypted_suffix: _unencrypted version: 3.12.1