From 5a0696ce644f1581c02c88ef890d0743f3f60771 Mon Sep 17 00:00:00 2001 From: yukkop Date: Mon, 8 Jun 2026 08:18:08 +0000 Subject: [PATCH] ci: `gitea`: runners infra --- .gitea/workflows/runner-nix-smoke.yaml | 29 + .gitea/workflows/runner-ubuntu-smoke.yaml | 31 ++ .sops.yaml | 9 + flake.lock | 8 +- infra/gitea-runners/image/README.md | 50 +- .../gitea-runners/k8s/cleanup-lifecycle.yaml | 154 ++++++ infra/gitea-runners/k8s/kustomization.yaml | 9 + infra/gitea-runners/k8s/namespace.yaml | 7 + infra/gitea-runners/k8s/runner-config.yaml | 36 ++ infra/gitea-runners/k8s/service-account.yaml | 36 ++ infra/gitea-runners/k8s/service.yaml | 16 + infra/gitea-runners/k8s/statefulset.yaml | 156 ++++++ infra/gitea-runners/opentofu/.gitignore | 29 + infra/gitea-runners/opentofu/README.md | 90 ++++ infra/gitea-runners/opentofu/backend.tf | 16 + infra/gitea-runners/opentofu/main.tf | 60 +++ infra/gitea-runners/opentofu/outputs.tf | 22 + infra/gitea-runners/opentofu/providers.tf | 1 + infra/gitea-runners/opentofu/variables.tf | 74 +++ infra/gitea-runners/opentofu/versions.tf | 17 + infra/gitea-runners/runbook.md | 503 ++++++++++++++++++ package/gitea-runner-nix-image/default.nix | 2 + sus/gitea-runners.yaml | 59 ++ 23 files changed, 1402 insertions(+), 12 deletions(-) create mode 100644 .gitea/workflows/runner-nix-smoke.yaml create mode 100644 .gitea/workflows/runner-ubuntu-smoke.yaml create mode 100644 infra/gitea-runners/k8s/cleanup-lifecycle.yaml create mode 100644 infra/gitea-runners/k8s/kustomization.yaml create mode 100644 infra/gitea-runners/k8s/namespace.yaml create mode 100644 infra/gitea-runners/k8s/runner-config.yaml create mode 100644 infra/gitea-runners/k8s/service-account.yaml create mode 100644 infra/gitea-runners/k8s/service.yaml create mode 100644 infra/gitea-runners/k8s/statefulset.yaml create mode 100644 infra/gitea-runners/opentofu/.gitignore create mode 100644 infra/gitea-runners/opentofu/README.md create mode 100644 infra/gitea-runners/opentofu/backend.tf create mode 100644 infra/gitea-runners/opentofu/main.tf create mode 100644 infra/gitea-runners/opentofu/outputs.tf create mode 100644 infra/gitea-runners/opentofu/providers.tf create mode 100644 infra/gitea-runners/opentofu/variables.tf create mode 100644 infra/gitea-runners/opentofu/versions.tf create mode 100644 infra/gitea-runners/runbook.md create mode 100644 sus/gitea-runners.yaml diff --git a/.gitea/workflows/runner-nix-smoke.yaml b/.gitea/workflows/runner-nix-smoke.yaml new file mode 100644 index 00000000..4016b15f --- /dev/null +++ b/.gitea/workflows/runner-nix-smoke.yaml @@ -0,0 +1,29 @@ +name: runner nix smoke + +on: + workflow_dispatch: + push: + branches: + - master + paths: + - .gitea/workflows/runner-nix-smoke.yaml + - flake.lock + - flake.nix + - infra/gitea-runners/** + +jobs: + smoke: + name: nix label smoke + runs-on: nix + steps: + - name: Nix version and cache configuration + run: | + set -eu + nix --version + nix config show substituters + nix config show trusted-public-keys + + - name: Repository flake evaluation + run: | + set -eu + nix flake check --no-build diff --git a/.gitea/workflows/runner-ubuntu-smoke.yaml b/.gitea/workflows/runner-ubuntu-smoke.yaml new file mode 100644 index 00000000..9ab1d848 --- /dev/null +++ b/.gitea/workflows/runner-ubuntu-smoke.yaml @@ -0,0 +1,31 @@ +name: runner ubuntu smoke + +on: + workflow_dispatch: + push: + branches: + - master + paths: + - .gitea/workflows/runner-ubuntu-smoke.yaml + - infra/gitea-runners/** + +jobs: + smoke: + name: ubuntu-latest label smoke + runs-on: ubuntu-latest + steps: + - name: Basic runner information + run: | + set -eu + echo "hello from gitea runner" + uname -a + + - name: Docker smoke when available + run: | + set -eu + if command -v docker >/dev/null 2>&1; then + docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}' + docker run --rm hello-world + else + echo "docker command not available; skipping Docker smoke" + fi diff --git a/.sops.yaml b/.sops.yaml index f81e2e75..58b43ea8 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -47,6 +47,15 @@ creation_rules: - *hectic-lab-server - *umbriel-bfs + - path_regex: sus/gitea-runners.yaml$ + key_groups: + - age: + - *nrv + - *yukkop + - *yukkop-alt + - *hectic-lab-server + - *umbriel-bfs + - path_regex: sus/matrix-cluster.yaml$ key_groups: - age: diff --git a/flake.lock b/flake.lock index 03c925c5..193d05cc 100644 --- a/flake.lock +++ b/flake.lock @@ -675,11 +675,11 @@ ] }, "locked": { - "lastModified": 1780868200, - "narHash": "sha256-h/u6QjA+GbOlKDMTkrZBQZrTf1oAnV9/i4+IsuGSKQg=", + "lastModified": 1780905541, + "narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=", "ref": "refs/heads/master", - "rev": "03956bafaea27ca89fe6e74a15658b2ae7547bec", - "revCount": 115, + "rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1", + "revCount": 123, "type": "git", "url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git" }, diff --git a/infra/gitea-runners/image/README.md b/infra/gitea-runners/image/README.md index 3f5aa537..df6e57b1 100644 --- a/infra/gitea-runners/image/README.md +++ b/infra/gitea-runners/image/README.md @@ -7,27 +7,61 @@ The repo-owned Nix-capable job image is built by the flake package nix build .#gitea-runner-nix-image ``` -The package currently emits a Docker archive tagged: +The package emits a Docker archive with the local build tag: ```text gitea-runner-nix-image:2026-06-07 ``` -After publishing to the internal registry, map the `nix` runner label to the -published, immutable image reference: +That tag is build metadata only. Do not use it as the final Gitea runner label +mapping because runner job images must be immutable. + +## Publication target + +Preferred registry: ```text -nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image:2026-06-07 +gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image ``` -Task 7 should replace the tag-only reference with the published digest once the -image is pushed, for example -`gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:`. +Publish the archive without adding secrets to the image layers, then use the +registry-reported digest as the only final `nix` label image reference: -Keep `ubuntu-latest` on the `gitea/runner` default image unless a later runner +```text +nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256: +``` + +The `2026-06-07` tag may be pushed as a human-readable companion tag, but the +runner label mapping must use the `@sha256:` reference above. Keep +`ubuntu-latest` on the `gitea/runner` default image unless a later runner configuration task explicitly changes it. Only the `nix` label should select this custom image. +If the Gitea container registry is unavailable, select a private registry that +is reachable from the runner Kubernetes cluster and requires authentication that +can be provided through Kubernetes image-pull secrets. Record the selected +registry and replace the host in the same digest-pinned form: + +```text +nix:docker:////gitea-runner-nix-image@sha256: +``` + +Do not fall back to `latest` or a tag-only mapping. + +## Task 7 publication status + +Local build evidence is recorded in +`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could +load and tag the image, but pushing to the preferred registry failed with +`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a +concrete final mapping. Kubernetes pull smoke is recorded in +`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl` +is not installed or not on `PATH`. + +Once registry credentials are available, rerun the push, capture the +registry-reported digest, and replace `` in the mapping above +before Task 6/9 consumes the label configuration. + ## Image contents The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its diff --git a/infra/gitea-runners/k8s/cleanup-lifecycle.yaml b/infra/gitea-runners/k8s/cleanup-lifecycle.yaml new file mode 100644 index 00000000..8bfb8b83 --- /dev/null +++ b/infra/gitea-runners/k8s/cleanup-lifecycle.yaml @@ -0,0 +1,154 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitea-runner-lifecycle + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner-lifecycle + app.kubernetes.io/part-of: gitea-actions +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitea-runner-lifecycle + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner-lifecycle + app.kubernetes.io/part-of: gitea-actions +rules: + - apiGroups: + - "" + resources: + - pods + - persistentvolumeclaims + verbs: + - get + - list + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitea-runner-lifecycle + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner-lifecycle + app.kubernetes.io/part-of: gitea-actions +subjects: + - kind: ServiceAccount + name: gitea-runner-lifecycle + namespace: gitea-runners +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitea-runner-lifecycle +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gitea-runner-lifecycle + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner-lifecycle + app.kubernetes.io/part-of: gitea-actions +data: + cleanup-dry-run.sh: | + #!/bin/sh + set -eu + + namespace="${RUNNER_NAMESPACE:-gitea-runners}" + mode="${CLEANUP_MODE:-dry-run}" + selector="app.kubernetes.io/name=gitea-runner" + + if [ "$namespace" != "gitea-runners" ]; then + printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2 + exit 13 + fi + + if [ "$mode" != "dry-run" ]; then + printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2 + exit 13 + fi + + printf 'gitea runner lifecycle cleanup dry-run\n' + printf 'namespace: %s\n' "$namespace" + printf 'mode: %s\n\n' "$mode" + + printf 'StatefulSet:\n' + kubectl -n "$namespace" get statefulset gitea-runner -o wide + + printf '\nActive runner pods:\n' + kubectl -n "$namespace" get pods -l "$selector" -o wide + + printf '\nRunner /data PVCs:\n' + kubectl -n "$namespace" get pvc -l "$selector" -o wide + + printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n' + found_candidate=0 + for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + pod="${pvc#data-}" + if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then + found_candidate=1 + printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod" + fi + done + + if [ "$found_candidate" -eq 0 ]; then + printf 'none\n' + fi + + printf '\nGitea registration reconciliation:\n' + printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n' + printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n' +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: gitea-runner-cleanup-dry-run + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner-lifecycle + app.kubernetes.io/part-of: gitea-actions +spec: + schedule: "17 3 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: gitea-runner-lifecycle + app.kubernetes.io/part-of: gitea-actions + spec: + serviceAccountName: gitea-runner-lifecycle + restartPolicy: Never + containers: + - name: cleanup-dry-run + image: bitnami/kubectl:1.30 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - /scripts/cleanup-dry-run.sh + env: + - name: RUNNER_NAMESPACE + value: gitea-runners + - name: CLEANUP_MODE + value: dry-run + volumeMounts: + - name: lifecycle-scripts + mountPath: /scripts + readOnly: true + volumes: + - name: lifecycle-scripts + configMap: + name: gitea-runner-lifecycle + defaultMode: 0555 diff --git a/infra/gitea-runners/k8s/kustomization.yaml b/infra/gitea-runners/k8s/kustomization.yaml new file mode 100644 index 00000000..829a2f49 --- /dev/null +++ b/infra/gitea-runners/k8s/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - service.yaml + - service-account.yaml + - runner-config.yaml + - statefulset.yaml + - cleanup-lifecycle.yaml diff --git a/infra/gitea-runners/k8s/namespace.yaml b/infra/gitea-runners/k8s/namespace.yaml new file mode 100644 index 00000000..bb4022ea --- /dev/null +++ b/infra/gitea-runners/k8s/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions diff --git a/infra/gitea-runners/k8s/runner-config.yaml b/infra/gitea-runners/k8s/runner-config.yaml new file mode 100644 index 00000000..e7d1a877 --- /dev/null +++ b/infra/gitea-runners/k8s/runner-config.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gitea-runner-config + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions +data: + config.yaml: | + log: + level: info + + runner: + file: /data/.runner + capacity: 1 + envs: {} + timeout: 3h + insecure: false + fetch_timeout: 5s + fetch_interval: 2s + labels: + - ubuntu-latest + # The nix label is intentionally disabled until the runner image has a + # concrete registry-reported digest; see ../runbook.md before deploy. + + cache: + enabled: true + dir: /data/cache + + container: + network: bridge + privileged: false + force_pull: true + valid_volumes: [] + docker_host: unix:///runner-docker/docker.sock diff --git a/infra/gitea-runners/k8s/service-account.yaml b/infra/gitea-runners/k8s/service-account.yaml new file mode 100644 index 00000000..b0dd437e --- /dev/null +++ b/infra/gitea-runners/k8s/service-account.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitea-runner + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitea-runner + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitea-runner + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions +subjects: + - kind: ServiceAccount + name: gitea-runner + namespace: gitea-runners +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitea-runner diff --git a/infra/gitea-runners/k8s/service.yaml b/infra/gitea-runners/k8s/service.yaml new file mode 100644 index 00000000..6725cc98 --- /dev/null +++ b/infra/gitea-runners/k8s/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitea-runner + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions +spec: + clusterIP: None + selector: + app.kubernetes.io/name: gitea-runner + ports: + - name: cache + port: 8088 + targetPort: cache diff --git a/infra/gitea-runners/k8s/statefulset.yaml b/infra/gitea-runners/k8s/statefulset.yaml new file mode 100644 index 00000000..e9d5bf79 --- /dev/null +++ b/infra/gitea-runners/k8s/statefulset.yaml @@ -0,0 +1,156 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gitea-runner + namespace: gitea-runners + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions +spec: + serviceName: gitea-runner + replicas: 5 + podManagementPolicy: Parallel + selector: + matchLabels: + app.kubernetes.io/name: gitea-runner + template: + metadata: + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions + annotations: + hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool." + spec: + serviceAccountName: gitea-runner + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: + fsGroup: 1000 + containers: + - name: runner + image: gitea/act_runner:0.2.11 + imagePullPolicy: IfNotPresent + env: + - name: GITEA_INSTANCE_URL + value: https://gitea.hectic-lab.com + - name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE + value: /runner-secrets/token + - name: CONFIG_FILE + value: /runner-config/config.yaml + - name: DOCKER_HOST + value: unix:///runner-docker/docker.sock + ports: + - name: cache + containerPort: 8088 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - test -s /data/.runner && test -S /runner-docker/docker.sock + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - test -S /runner-docker/docker.sock + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /runner-config + readOnly: true + - name: runner-token + mountPath: /runner-secrets + readOnly: true + - name: docker-socket + mountPath: /runner-docker + - name: docker + image: docker:27-dind + imagePullPolicy: IfNotPresent + args: + - --host=unix:///runner-docker/docker.sock + - --storage-driver=overlay2 + - --tls=false + env: + - name: DOCKER_TLS_CERTDIR + value: "" + - name: DOCKER_HOST + value: unix:///runner-docker/docker.sock + securityContext: + # Privileged rootful DinD is intentionally scoped to this trusted + # internal runner pool; never expose it to untrusted fork/PR jobs. + privileged: true + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi + livenessProbe: + exec: + command: + - docker + - info + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - docker + - info + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 6 + volumeMounts: + - name: docker-socket + mountPath: /runner-docker + - name: docker-graph + mountPath: /var/lib/docker + volumes: + - name: config + configMap: + name: gitea-runner-config + - name: runner-token + secret: + secretName: gitea-runner-token + items: + - key: token + path: token + defaultMode: 0400 + - name: docker-socket + emptyDir: {} + - name: docker-graph + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: gitea-runner + app.kubernetes.io/part-of: gitea-actions + spec: + accessModes: + - ReadWriteOnce + storageClassName: hcloud-volumes + resources: + requests: + storage: 20Gi diff --git a/infra/gitea-runners/opentofu/.gitignore b/infra/gitea-runners/opentofu/.gitignore new file mode 100644 index 00000000..690eee92 --- /dev/null +++ b/infra/gitea-runners/opentofu/.gitignore @@ -0,0 +1,29 @@ +# OpenTofu working directory and downloaded modules/providers. +.terraform/ +.terraform.lock.hcl + +# State must live in the S3 backend for production. Local state is allowed only +# for throwaway syntax checks with `tofu init -backend=false` and must not be +# committed. +terraform.tfstate +terraform.tfstate.* +*.tfstate +*.tfstate.* +crash.log +crash.*.log + +# Plans can contain secrets or derived infrastructure data. +*.tfplan +*.plan +kubeconfig +kubeconfig.yaml +*_kubeconfig.yaml + +# Variable files commonly carry credentials. Keep production inputs in SOPS or +# external environment/configuration, not in checked-in files. +*.tfvars +*.tfvars.json +override.tf +override.tf.json +*_override.tf +*_override.tf.json diff --git a/infra/gitea-runners/opentofu/README.md b/infra/gitea-runners/opentofu/README.md new file mode 100644 index 00000000..ed56d608 --- /dev/null +++ b/infra/gitea-runners/opentofu/README.md @@ -0,0 +1,90 @@ +# Gitea runner OpenTofu backend contract + +This directory defines the safe backend, provider contract, and kube-hetzner +cluster stack for the Gitea runner Kubernetes cluster. + +## Required backend + +Production state must use the OpenTofu S3 backend in `backend.tf`: + +- bucket: `gitea-runner-hectic-lab` +- key: `gitea-runners/kube-hetzner/terraform.tfstate` +- region: `fsn1`, aligned with the target Hetzner location +- encryption: `encrypt = true` +- locking: `use_lockfile = true` where the selected S3-compatible endpoint + supports it + +Before any production `tofu init`, verify the S3-compatible endpoint, credential +source, bucket versioning, encryption behavior, and lockfile support for the +chosen object-storage provider. Keep backend authentication externalized through +environment variables, AWS-compatible shared config, or the production secret +injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner +tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in +OpenTofu files. + +## Local state safety + +Production local state is forbidden. Only syntax-only validation/prototyping may +use local state, and it must use backend-disabled initialization: + +```sh +tofu -chdir=infra/gitea-runners/opentofu init -backend=false +tofu -chdir=infra/gitea-runners/opentofu validate +``` + +Fail the run if production local state appears: + +```sh +test ! -e infra/gitea-runners/opentofu/terraform.tfstate +test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup +grep -R 'backend "s3"' infra/gitea-runners/opentofu +``` + +The `.gitignore` in this directory blocks local state, plans, downloaded +providers/modules, and variable files from being committed. Treat any local +state file as disposable validation residue, never as production state. + +## Provider and module pins + +`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to +`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan +observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and +module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded +as locals so Task 5 can wire the module without re-opening the version contract. + +`providers.tf` leaves the `hcloud` provider empty so authentication comes from +the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`. +Do not set token values in `.tf` or `.tfvars` files. + +## Cluster shape + +The default cluster is deliberately fixed-size: + +- cluster name: `gitea-runners` +- Hetzner location: `fsn1` +- private network region: `eu-central` +- control plane: one `cpx21` node in pool `control-plane` +- workers: three `cpx31` nodes in pool `runner-workers` +- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes` +- Longhorn: disabled +- autoscaling/KEDA: not enabled in this stack + +The three default workers are sized for the initial five trusted privileged DinD +jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise +`worker_count` to `5` or increase `worker_server_type`, then run a fresh +`tofu plan` and the Task 11 Kubernetes pressure checks before applying. + +Required inputs must come from environment or secret injection, for example +`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`. +Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated +kubeconfig to `./_kubeconfig.yaml` when `create_kubeconfig` is +enabled; this path is ignored as operational secret material. + +## Known state caveat + +kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its +internal `kube_system_secrets` handling. This task does not claim that risk is +solved. Task 5 must verify the generated plan and state before production apply +and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private +keys, and decrypted secrets are absent from committed files and unsafe state +evidence. diff --git a/infra/gitea-runners/opentofu/backend.tf b/infra/gitea-runners/opentofu/backend.tf new file mode 100644 index 00000000..47e95466 --- /dev/null +++ b/infra/gitea-runners/opentofu/backend.tf @@ -0,0 +1,16 @@ +terraform { + backend "s3" { + bucket = "gitea-runner-hectic-lab" + key = "gitea-runners/kube-hetzner/terraform.tfstate" + region = "fsn1" + encrypt = true + use_lockfile = true + } +} + +check "remote_state_contract" { + assert { + condition = local.production_remote_state + error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden." + } +} diff --git a/infra/gitea-runners/opentofu/main.tf b/infra/gitea-runners/opentofu/main.tf new file mode 100644 index 00000000..af41f0af --- /dev/null +++ b/infra/gitea-runners/opentofu/main.tf @@ -0,0 +1,60 @@ +locals { + default_storage_class = "hcloud-volumes" + + control_plane_nodepools = [ + { + name = "control-plane" + server_type = var.control_plane_server_type + location = var.hetzner_location + labels = [] + taints = [] + count = 1 + }, + ] + + agent_nodepools = [ + { + name = "runner-workers" + server_type = var.worker_server_type + location = var.hetzner_location + labels = ["node-role.hectic-lab/gitea-runner=true"] + taints = [] + count = var.worker_count + }, + ] +} + +module "kube_hetzner" { + source = "kube-hetzner/kube-hetzner/hcloud" + version = "2.19.3" + + providers = { + hcloud = hcloud + } + + hcloud_token = var.hcloud_token + ssh_public_key = var.ssh_public_key + ssh_private_key = var.ssh_private_key + + cluster_name = var.cluster_name + base_domain = var.base_domain + + # kube-hetzner v2.19.3 writes _kubeconfig.yaml; outputs below + # expose that expected path without outputting kubeconfig private key material. + create_kubeconfig = true + + network_region = var.network_region + load_balancer_location = var.hetzner_location + control_plane_nodepools = local.control_plane_nodepools + agent_nodepools = local.agent_nodepools + + # Hetzner CSI is the required StorageClass provider for runner PVCs. + disable_hetzner_csi = false + + # Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only. + enable_longhorn = false + + # Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and + # raise worker_count to 5 or increase worker_server_type after validating pod + # CPU, memory, and ephemeral-storage pressure in Task 11. +} diff --git a/infra/gitea-runners/opentofu/outputs.tf b/infra/gitea-runners/opentofu/outputs.tf new file mode 100644 index 00000000..05bf996b --- /dev/null +++ b/infra/gitea-runners/opentofu/outputs.tf @@ -0,0 +1,22 @@ +output "kubeconfig_path" { + description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed." + value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml") +} + +output "cluster_name" { + description = "kube-hetzner cluster name." + value = var.cluster_name +} + +output "node_pool_names" { + description = "Control-plane and worker node pool names used by this stack." + value = { + control_plane = [for pool in local.control_plane_nodepools : pool.name] + workers = [for pool in local.agent_nodepools : pool.name] + } +} + +output "default_storage_class" { + description = "Default Hetzner CSI StorageClass expected for runner PVCs." + value = local.default_storage_class +} diff --git a/infra/gitea-runners/opentofu/providers.tf b/infra/gitea-runners/opentofu/providers.tf new file mode 100644 index 00000000..988ef060 --- /dev/null +++ b/infra/gitea-runners/opentofu/providers.tf @@ -0,0 +1 @@ +provider "hcloud" {} diff --git a/infra/gitea-runners/opentofu/variables.tf b/infra/gitea-runners/opentofu/variables.tf new file mode 100644 index 00000000..40b5c668 --- /dev/null +++ b/infra/gitea-runners/opentofu/variables.tf @@ -0,0 +1,74 @@ +variable "hcloud_token" { + description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply." + type = string + sensitive = true +} + +variable "ssh_public_key" { + description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path." + type = string +} + +variable "ssh_private_key" { + description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it." + type = string + sensitive = true +} + +variable "cluster_name" { + description = "Name for the kube-hetzner runner cluster." + type = string + default = "gitea-runners" + + validation { + condition = can(regex("^[a-z0-9-]+$", var.cluster_name)) + error_message = "cluster_name must contain only lowercase letters, numbers, and dashes." + } +} + +variable "hetzner_location" { + description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein." + type = string + default = "fsn1" +} + +variable "network_region" { + description = "Hetzner private network region. eu-central covers fsn1." + type = string + default = "eu-central" +} + +variable "control_plane_server_type" { + description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads." + type = string + default = "cpx21" +} + +variable "worker_server_type" { + description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation." + type = string + default = "cpx31" +} + +variable "worker_count" { + description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack." + type = number + default = 3 + + validation { + condition = var.worker_count >= 1 + error_message = "worker_count must be at least 1." + } +} + +variable "kubeconfig_path" { + description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as _kubeconfig.yaml when create_kubeconfig is true." + type = string + default = null +} + +variable "base_domain" { + description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults." + type = string + default = "" +} diff --git a/infra/gitea-runners/opentofu/versions.tf b/infra/gitea-runners/opentofu/versions.tf new file mode 100644 index 00000000..376ab4e0 --- /dev/null +++ b/infra/gitea-runners/opentofu/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.10.1" + + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "1.60.1" + } + } +} + +locals { + kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud" + kube_hetzner_module_version = "2.19.3" + hcloud_provider_minimum = ">= 1.59.0" + production_remote_state = true +} diff --git a/infra/gitea-runners/runbook.md b/infra/gitea-runners/runbook.md new file mode 100644 index 00000000..8fdfda22 --- /dev/null +++ b/infra/gitea-runners/runbook.md @@ -0,0 +1,503 @@ +# Gitea Runner Infrastructure Runbook + +## Scope + +This directory is the repo-owned boundary for the first Gitea Actions runner +pool. Task 1 only establishes the scaffold and immutable decision contract; +downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests, +and a Nix-capable runner image under the existing subdirectories. + +The target service is `https://gitea.hectic-lab.com` for the Gitea organization +`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means +workflow job containers are ephemeral, while each runner pod keeps its runner +identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC. + +## Immutable decisions + +- Infrastructure is managed with OpenTofu command examples only, using the + `tofu` CLI. +- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner. +- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`. +- Runner implementation is the non-Enterprise `gitea/runner`. +- Runner registration uses a Gitea organization-scoped token for `hectic-lab`. +- Runtime token delivery is SOPS-backed and mounted into the runner pod as a + file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token + environment variables are not the contract. +- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for + `/data`, including `/data/.runner`. +- Container builds run through privileged rootful DinD inside trusted runner + pods; host Docker socket mounting is not an implementation path. +- The active runner label is `ubuntu-latest`. The `nix` label is not live until + the Nix-capable image has been pushed and a concrete registry-reported digest + is added to the runner ConfigMap. +- First scope is trusted internal workflows only, with no untrusted fork or PR + workflow support. +- First scope has no autoscaling, no KEDA, and no dynamic runner controller. + +## Lifecycle boundaries + +- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend + contract, Hetzner provider configuration, and kube-hetzner module wiring. +- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount, + StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work. +- `infra/gitea-runners/image/`: downstream notes or sources for the runner image + handoff; package or flake output changes are outside Task 1. +- `infra/gitea-runners/runbook.md`: this contract plus later operational + commands, rollback notes, and acceptance evidence references. + +## Guardrails + +- Enterprise ARC/actions-runner-controller are rejected alternatives and must + not be implemented here. Do not add ARC custom resources, controller install + instructions, or GitHub Actions ARC assumptions. +- Untrusted fork/PR workflows are out of first scope; privileged DinD is only + acceptable for trusted internal jobs. +- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet + runner pool. +- No actual secrets are committed: no kubeconfig, runner token, Hetzner token, + S3 credentials, decrypted SOPS files, or SOPS age keys. +- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea + runner token; Kubernetes receives the token as a mounted file secret instead. +- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers; + jobs must reach the public HTTPS service. + +## Initial acceptance commands + +Run from the repository root: + +```sh +test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image +test -f infra/gitea-runners/runbook.md +grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md +grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners +grep -R "[t]erraform " infra/gitea-runners || true +grep -R "[D]ECISION NEEDED" infra/gitea-runners || true +``` + +Expected outcomes: the directory and file checks exit 0; the architecture-term +grep shows this contract; ARC references appear only in the rejected-alternative +guardrail above; there are no forbidden CLI command examples and no unresolved +decision placeholders. + +## Downstream placeholders + +- Task 2: add OpenTofu backend/provider files and verify S3 state safety. +- Task 3: add SOPS secret contract and runtime token delivery details. +- Task 4: define or package the Nix-capable runner image for the `nix` label. +- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows, + and document cleanup, rollback, and scaling operations. + +## Runner lifecycle cleanup + +All lifecycle commands are scoped to the runner namespace: + +```sh +kubectl -n gitea-runners get statefulset gitea-runner +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide +``` + +The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active +runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It +does not delete pods, PVCs, Docker data, or Gitea runner registrations. + +Run the same inventory on demand without waiting for the schedule: + +```sh +kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run +kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m +kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run +``` + +The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed +manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an +operator to remove finished jobs manually. + +If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching +pod loses `/data/.runner`. That runner identity must then be deregistered from +Gitea or the replacement pod must be allowed to re-register intentionally with +the current organization runner token. Do not delete an active runner PVC as a +normal cleanup step. + +Non-UI Gitea registration reconciliation uses the Gitea API with a separate +admin token. Store that token outside this repository and pass it as a file; do +not print it: + +```sh +kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token +kubectl -n gitea-runners run gitea-runner-registration-dry-run \ + --restart=Never \ + --image=curlimages/curl:8.10.1 \ + --overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}' +kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run +``` + +Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod +has completed and its logs have been collected. Do not keep this admin token in +the runner namespace longer than the reconciliation window. + +Only remove a stale Gitea runner registration after the corresponding pod/PVC +was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a +Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI +cleanup is a fallback, not the only path. Record the removed runner name and the +Kubernetes PVC/pod deletion that made it stale. + +After the dry-run list identifies a stale registration and the PVC/pod deletion +has been recorded, remove that exact Gitea runner by id through the API: + +```sh +runner_id='REPLACE_WITH_STALE_RUNNER_ID' +umask 077 +curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX) +trap 'rm -f "$curl_config"' EXIT +{ + printf 'request = "DELETE"\n' + printf 'header = "Authorization: token ' + cat /secure/path/gitea-admin-token + printf '"\n' + printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id" +} > "$curl_config" +curl -fsS --config "$curl_config" +``` + +Do not run the delete command for a runner that still has an active +`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is +being intentionally reset for re-registration. + +## Docker-in-Docker storage cleanup + +Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the +pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always +list disk usage before pruning, and run the command only against the `docker` +container in runner pods in `gitea-runners`. Because this storage is pod-local, +loop over pods for pool-wide cleanup: + +```sh +for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df +done + +for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h +done + +for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df +done +``` + +For one pod, replace the StatefulSet target with the pod name: + +```sh +kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df +kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h +``` + +Do not run host-level Docker cleanup commands and do not mount or prune a host +Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by +Kubernetes; the `/data` PVC remains and still controls runner identity. + +## Token rotation + +Rotate the Gitea organization runner token without printing decrypted values: + +```sh +sops sus/gitea-runners.yaml +umask 077 +token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX) +trap 'rm -f "$token_file"' EXIT +sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file" +kubectl -n gitea-runners create secret generic gitea-runner-token \ + --from-file=token="$token_file" \ + --dry-run=client \ + -o yaml | kubectl -n gitea-runners apply -f - +kubectl -n gitea-runners rollout restart statefulset/gitea-runner +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true +``` + +The `rollout restart` command above is the controlled restart path for this +StatefulSet. Observe the rollout and each ordinal until all replacement pods are +Ready; do not delete runner pods directly as part of normal token rotation: + +```sh +kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m +kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m +kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m +kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m +kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +``` + +Verification must confirm the token file mount remains present while the token +value never appears in logs or evidence: + +```sh +kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token' +kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true +``` + +## Deploy and status + +These commands are executable only when the external inputs are available: + +- `TF_VAR_hcloud_token` +- `TF_VAR_ssh_public_key` +- `TF_VAR_ssh_private_key` +- S3 backend credentials and endpoint access +- a matching SOPS age identity for `sus/gitea-runners.yaml` +- `kubectl` access to the target cluster +- a concrete digest for the pushed Nix-capable runner image, if enabling the + `nix` label + +If any input is missing, stop before `tofu apply`. Do not guess values or reuse +stale kubeconfig files. + +Before production Kubernetes apply or rollout, satisfy both manifest gates: + +1. Create or update the `gitea-runner-token` Secret from SOPS. The active + Kustomize overlay intentionally does not include a placeholder Secret, but + the StatefulSet still mounts `secretName: gitea-runner-token` as + `/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`. +2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable + image has been pushed successfully. Enable the `nix` label only by adding a + digest-pinned `docker://` mapping with the exact registry-reported sha256 + digest from that push. + +Use the same SOPS materialization pattern as token rotation before applying the +Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a +target namespace; the full overlay remains gated on the Secret and digest +decisions: + +```sh +kubectl apply -f infra/gitea-runners/k8s/namespace.yaml +umask 077 +token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX) +trap 'rm -f "$token_file"' EXIT +sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file" +kubectl -n gitea-runners create secret generic gitea-runner-token \ + --from-file=token="$token_file" \ + --dry-run=client \ + -o yaml | kubectl -n gitea-runners apply -f - +``` + +Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command +above succeeds. Do not claim or enable the `nix` runner label until the image +publication step has produced the concrete digest. + +```sh +tofu -chdir=infra/gitea-runners/opentofu init +tofu -chdir=infra/gitea-runners/opentofu validate +tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan +tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan +export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)" +kubectl config current-context +kubectl get nodes -o wide +kubectl get sc +kubectl apply -k infra/gitea-runners/k8s +kubectl -n gitea-runners get statefulset gitea-runner +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50 +kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 +``` + +Expected status after deploy: + +- `kubectl config current-context` names the runner cluster context. +- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready. +- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs. +- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas. +- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs. +- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value. + +## Scale 5 to 10 to 5 + +Scaling is a temporary capacity exercise, not the steady-state setting. Scale up, +wait for readiness, run the concurrent smoke jobs, then scale back down to 5. + +```sh +kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10 +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide + +# Run the concurrent smoke workflows now. + +kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5 +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide +``` + +After scaling back down, inspect the cleanup dry-run and deregister any stale +runner registrations only for pods or PVCs that were intentionally removed. + +## Cleanup and stale runner deregistration + +Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any +PVC candidates whose pod is gone. It must not delete active resources. + +```sh +kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run +kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m +kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run +``` + +Pool-wide DinD storage checks and cleanup: + +```sh +for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df +done + +for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h +done + +for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df +done +``` + +If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching +pod loses `/data/.runner`. Deregister that runner from Gitea, or let the +replacement pod re-register intentionally with the current organization token. +Never delete an active runner PVC as routine cleanup. + +Non-UI Gitea registration reconciliation uses an admin token stored outside this +repository: + +```sh +kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token +kubectl -n gitea-runners run gitea-runner-registration-dry-run \ + --restart=Never \ + --image=curlimages/curl:8.10.1 \ + --overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}' +kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run +``` + +Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run +pod has completed and its logs have been collected. + +After the dry-run list identifies a stale registration and the PVC or pod +deletion has been recorded, remove that exact Gitea runner by id through the +API: + +```sh +runner_id='REPLACE_WITH_STALE_RUNNER_ID' +umask 077 +curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX) +trap 'rm -f "$curl_config"' EXIT +{ + printf 'request = "DELETE"\n' + printf 'header = "Authorization: token ' + cat /secure/path/gitea-admin-token + printf '"\n' + printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id" +} > "$curl_config" +curl -fsS --config "$curl_config" +``` + +Do not run the delete command for a runner that still has an active +`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is +being intentionally reset for re-registration. + +## Application rollback + +Rollback the app layer only. Do not use this section to destroy the cluster. + +```sh +kubectl -n gitea-runners rollout history statefulset/gitea-runner +kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision= +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide +kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 +``` + +If a manifest rollback is needed, reapply the repo overlay after checking out the +known-good revision, then re-run the rollout checks: + +```sh +kubectl -n gitea-runners apply -k infra/gitea-runners/k8s +kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m +``` + +## Full cluster teardown + +This destroys Hetzner resources owned by the kube-hetzner stack, including the +`gitea-runners` cluster nodes, the `control-plane` node pool, the +`runner-workers` node pool, the cluster network, load balancer resources, +firewall objects, and any attached Hetzner CSI volumes still managed by the +stack. Do not run teardown unless the destruction is intentional. + +```sh +tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan +tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan +tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan +``` + +## Partial OpenTofu apply recovery + +If `tofu apply` fails after creating some resources, do not destroy blindly. +First reconcile state and inspect what the stack thinks exists: + +```sh +tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan +tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan +tofu -chdir=infra/gitea-runners/opentofu state list +``` + +Then rerun the normal plan path. Use `-target` only as a last resort when a +single resource is stuck and the drift is understood. + +## S3 backend recovery + +If backend init or state access fails, first verify the bucket and versioning +outside OpenTofu, then reconfigure the backend: + +```sh +nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab +nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab +tofu -chdir=infra/gitea-runners/opentofu init -reconfigure +tofu -chdir=infra/gitea-runners/opentofu plan +``` + +If the backend reports a stale lock, confirm no `tofu` process is active, then +use `tofu force-unlock ` with the lock id from the error. Never force +unlock a live plan or apply. + +## Gitea outage troubleshooting + +Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job +containers. + +```sh +kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz +kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true +kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50 +``` + +If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete +`/data/.runner` just because the service is unavailable. Once Gitea returns, +repeat the token rotation or re-registration path if a pod restarted while the +service was unavailable and lost its runner identity. + +## Release checklist + +Do not release unless the following evidence files exist and are readable: + +- `.sisyphus/evidence/task-5-cluster-plan.txt` +- `.sisyphus/evidence/task-5-secret-plan-scan.txt` +- `.sisyphus/evidence/task-9-deploy.txt` +- `.sisyphus/evidence/task-9-secret-mount.txt` +- `.sisyphus/evidence/task-10-ubuntu-workflow.txt` +- `.sisyphus/evidence/task-10-nix-workflow.txt` +- `.sisyphus/evidence/task-11-scale.txt` +- `.sisyphus/evidence/task-11-restart-cleanup.txt` + +If any evidence file is missing, stop and collect it before treating the runbook +as complete. diff --git a/package/gitea-runner-nix-image/default.nix b/package/gitea-runner-nix-image/default.nix index ad145b79..c0c304cf 100644 --- a/package/gitea-runner-nix-image/default.nix +++ b/package/gitea-runner-nix-image/default.nix @@ -53,9 +53,11 @@ dockerTools.buildLayeredImageWithNixDb { chmod 1777 tmp cat > etc/nix/nix.conf <<'EOF' + accept-flake-config = true experimental-features = nix-command flakes substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA= + trusted-users = root sandbox = false EOF ''; diff --git a/sus/gitea-runners.yaml b/sus/gitea-runners.yaml new file mode 100644 index 00000000..d92b496d --- /dev/null +++ b/sus/gitea-runners.yaml @@ -0,0 +1,59 @@ +gitea: + hectic-lab: + org-runner-registration-token: ENC[AES256_GCM,data:ouupBDavLapNezKtcIViqfhsiQJTXwOL3eE9lLDutIvxk+oxOxUmbO4=,iv:Bck5ZYbDlYnBADSLk4lLtskzX3bqFCVUzj6AbWWJXBU=,tag:8y8nHDELNvlDEkrw0bwXKw==,type:str] +hetzner: + hcloud-token: ENC[AES256_GCM,data:uvqCdk+0S3HhSeo/NlGj42ii6Q9aCKq6r0XLuoelBJA=,iv:0sDmmulU4IWIqvhS9n2BTGUutuvZq+8kqW79nBaShBk=,tag:qnQxz5j8sTP9JA5hNFWliw==,type:str] +s3: + access-key-id: ENC[AES256_GCM,data:fzYW2dI2q43lx7Mb1vMzDAGQeps65ro/0XroLA==,iv:bCmlH4J0Jo0ADSgQ+XxlB4Pjk+1pkmILzYbpfHGyaEs=,tag:m5YtDNOj8u7F1OVtM9NLvQ==,type:str] + secret-access-key: ENC[AES256_GCM,data:wxODth7RJNQsmbqr3stAclVAsoeU2BbY94GfkSyORzs=,iv:9QUOyK/2Kc2VuIylsT5d7WOQIiUdwpuzTNvwTW02JBg=,tag:7oY4p1dMzB1A3AeCYYmswA==,type:str] +sops: + age: + - recipient: age1x04u7ftjgx8de2gq596e7frauze764cmn7jjwqnx8szthvfft5qq0tezx6 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNjZxVlMwUkpuU3lOZkxZ + SlQwRk5vQmRQdWprS20zSmJncjBpMExjVkdzCk9RV2lrTjE4UXZPMGtXK0NJMEVR + cDhDOXNpL1NYNHpKais1VEs0Si9EOG8KLS0tIFo2QmpFMEtYQVBVYjdpOEorblRr + L3l6MjNZdHZHblZJejdmRXpVUWIxb0UK4xP/kIxCge5t0XGv70BUt4oLUGHB/6Pj + 2bX+gUq3KMzgEJahOgN67EJBJj7Ixs9B+MpBZkmTE9WddvyHyXffwQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1r25zdeqq8nac6dgca9en28r57ffyz9u9d8z5yc25gc8xqz747vaqmdtk0h + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTkpzdUo5V2FzWHlWbkFi + VmhtNWhoR1pCd2RRWWJxYlV4YVRvSThmSXlFCmtuVDlFd1lJa21xaDFTQ3JhazZB + RkM1ZkRyS0VGK1VUajFXeEozZEZYMmMKLS0tIHRwT2JiNGJmYjdvUGpRT3o2cTNC + UTFIbFZwYXo0T0REQkRNT3ZLS0VjRjgKCompPQJCfhxF/fNYT909szq5KDVDkzzB + 9CKFoGVRYk+5cbBk1zprGuqoBCWEJ85MYKjueEJKowxlCb+8/rr/Ug== + -----END AGE ENCRYPTED FILE----- + - recipient: age1vv46vn4hsn2lg6jy834cpu40c3mvqklldcm3hjtynrhwtpmlpc8szruz4v + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPMzM3bExWZlRVRHl3eHg5 + ai9SUTU3YjRodm90T2poc2dJRTkzcWJic1RRCmJMVytUK0pQWk5KVTJNT0RSZUUz + TS8xRHNLQjVZM2RGY0JjTXJrMjFrSlUKLS0tIE9nd29kNXRhK2xKVlEvR29HbFht + ZU1jOGprOXhZTC9HZ0RUZFpZeFV6eGsKhw+TXpiEd7dYPG84xjr3rV/dPm0Nilgl + 509h/L3qqp7pJUOjZWidMB63dr1/DlTxjRIKubBy0eqidCcnkDnDfQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age13h8twnwvgxn04l5ywtru89a6psw5d0uckr2eghxsjp88a5augvsstq5ard + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFbjF2ZkpKUlZFRE9CUEw0 + Y3kvRzM0blNHdkFEMlFxdHdNQjY1cFdKWjA4CkNPdHZnM3NqQXZId05UQWFPQ1B1 + d0RrM2lBb3JNMlBNV2VCTFhPS3BHMEUKLS0tIHpNMEt3WnJlMWJVWHIzbjdVdkFL + WklFVDlBOWY4Kzd6TzlOKzBPNmtobHMKw0zroE9HR0yJPebYPpCJ0ubB0WSPagQ9 + u01s9TGMYtxYsD/WOCo/gJSGbilixl4RbmQzpxfzklco2ht73M1q3Q== + -----END AGE ENCRYPTED FILE----- + - recipient: age1jxntjca8q2vxvf2jaal4xyvm2ae6sh62fhv897694kuzawfrk5asj00zdt + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZTQ4dk9OdlVWaVVGdm12 + QXVVVlpUaFFOOTByQVozWVJYdXdBRitsZHlFCkxyWlhqR0NaKzd0Y3lJcFg3eGZB + ejBMbnVvZXFvUkxsRitvQ0N3Vnh6dWsKLS0tIHM1aG5zTGE3QVZSdEdNbGtvekxB + UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P + MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-06-07T22:55:43Z" + mac: ENC[AES256_GCM,data:+FNn3iLTgWF5470FCDus3uBpSir3E0nyajtwAjTj4yoVBaGk1QcFqDxA+S9wD85Ch050uHyeiR+JlApeeNltZtnidPIStQdj5tdP2UP+b2v8se4UzQyPCWTPN0wRIZ9JMubRhErZlCGovJlr5UHYdg/+gFfICPOthmFeCqEf5tM=,iv:Vq64SW0srhiappVknbdTtqichtsUDB0LeBlns43dO4Q=,tag:pVA0K2i3pP2Cpo1VSOf8dQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.10.2