ci: gitea: runners infra
This commit is contained in:
@@ -0,0 +1,29 @@
|
|||||||
|
name: runner nix smoke
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- .gitea/workflows/runner-nix-smoke.yaml
|
||||||
|
- flake.lock
|
||||||
|
- flake.nix
|
||||||
|
- infra/gitea-runners/**
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: nix label smoke
|
||||||
|
runs-on: nix
|
||||||
|
steps:
|
||||||
|
- name: Nix version and cache configuration
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
nix --version
|
||||||
|
nix config show substituters
|
||||||
|
nix config show trusted-public-keys
|
||||||
|
|
||||||
|
- name: Repository flake evaluation
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
nix flake check --no-build
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
name: runner ubuntu smoke
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- .gitea/workflows/runner-ubuntu-smoke.yaml
|
||||||
|
- infra/gitea-runners/**
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: ubuntu-latest label smoke
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Basic runner information
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
echo "hello from gitea runner"
|
||||||
|
uname -a
|
||||||
|
|
||||||
|
- name: Docker smoke when available
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
|
||||||
|
docker run --rm hello-world
|
||||||
|
else
|
||||||
|
echo "docker command not available; skipping Docker smoke"
|
||||||
|
fi
|
||||||
@@ -47,6 +47,15 @@ creation_rules:
|
|||||||
- *hectic-lab-server
|
- *hectic-lab-server
|
||||||
- *umbriel-bfs
|
- *umbriel-bfs
|
||||||
|
|
||||||
|
- path_regex: sus/gitea-runners.yaml$
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *nrv
|
||||||
|
- *yukkop
|
||||||
|
- *yukkop-alt
|
||||||
|
- *hectic-lab-server
|
||||||
|
- *umbriel-bfs
|
||||||
|
|
||||||
- path_regex: sus/matrix-cluster.yaml$
|
- path_regex: sus/matrix-cluster.yaml$
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
|
|||||||
Generated
+4
-4
@@ -675,11 +675,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1780868200,
|
"lastModified": 1780905541,
|
||||||
"narHash": "sha256-h/u6QjA+GbOlKDMTkrZBQZrTf1oAnV9/i4+IsuGSKQg=",
|
"narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
|
||||||
"ref": "refs/heads/master",
|
"ref": "refs/heads/master",
|
||||||
"rev": "03956bafaea27ca89fe6e74a15658b2ae7547bec",
|
"rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
|
||||||
"revCount": 115,
|
"revCount": 123,
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -7,27 +7,61 @@ The repo-owned Nix-capable job image is built by the flake package
|
|||||||
nix build .#gitea-runner-nix-image
|
nix build .#gitea-runner-nix-image
|
||||||
```
|
```
|
||||||
|
|
||||||
The package currently emits a Docker archive tagged:
|
The package emits a Docker archive with the local build tag:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
gitea-runner-nix-image:2026-06-07
|
gitea-runner-nix-image:2026-06-07
|
||||||
```
|
```
|
||||||
|
|
||||||
After publishing to the internal registry, map the `nix` runner label to the
|
That tag is build metadata only. Do not use it as the final Gitea runner label
|
||||||
published, immutable image reference:
|
mapping because runner job images must be immutable.
|
||||||
|
|
||||||
|
## Publication target
|
||||||
|
|
||||||
|
Preferred registry:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image:2026-06-07
|
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
|
||||||
```
|
```
|
||||||
|
|
||||||
Task 7 should replace the tag-only reference with the published digest once the
|
Publish the archive without adding secrets to the image layers, then use the
|
||||||
image is pushed, for example
|
registry-reported digest as the only final `nix` label image reference:
|
||||||
`gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<digest>`.
|
|
||||||
|
|
||||||
Keep `ubuntu-latest` on the `gitea/runner` default image unless a later runner
|
```text
|
||||||
|
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
```
|
||||||
|
|
||||||
|
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
|
||||||
|
runner label mapping must use the `@sha256:` reference above. Keep
|
||||||
|
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
|
||||||
configuration task explicitly changes it. Only the `nix` label should select
|
configuration task explicitly changes it. Only the `nix` label should select
|
||||||
this custom image.
|
this custom image.
|
||||||
|
|
||||||
|
If the Gitea container registry is unavailable, select a private registry that
|
||||||
|
is reachable from the runner Kubernetes cluster and requires authentication that
|
||||||
|
can be provided through Kubernetes image-pull secrets. Record the selected
|
||||||
|
registry and replace the host in the same digest-pinned form:
|
||||||
|
|
||||||
|
```text
|
||||||
|
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not fall back to `latest` or a tag-only mapping.
|
||||||
|
|
||||||
|
## Task 7 publication status
|
||||||
|
|
||||||
|
Local build evidence is recorded in
|
||||||
|
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
|
||||||
|
load and tag the image, but pushing to the preferred registry failed with
|
||||||
|
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
|
||||||
|
concrete final mapping. Kubernetes pull smoke is recorded in
|
||||||
|
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
|
||||||
|
is not installed or not on `PATH`.
|
||||||
|
|
||||||
|
Once registry credentials are available, rerun the push, capture the
|
||||||
|
registry-reported digest, and replace `<registry-digest>` in the mapping above
|
||||||
|
before Task 6/9 consumes the label configuration.
|
||||||
|
|
||||||
## Image contents
|
## Image contents
|
||||||
|
|
||||||
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
|
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- persistentvolumeclaims
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- apiGroups:
|
||||||
|
- apps
|
||||||
|
resources:
|
||||||
|
- statefulsets
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
data:
|
||||||
|
cleanup-dry-run.sh: |
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
namespace="${RUNNER_NAMESPACE:-gitea-runners}"
|
||||||
|
mode="${CLEANUP_MODE:-dry-run}"
|
||||||
|
selector="app.kubernetes.io/name=gitea-runner"
|
||||||
|
|
||||||
|
if [ "$namespace" != "gitea-runners" ]; then
|
||||||
|
printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
|
||||||
|
exit 13
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$mode" != "dry-run" ]; then
|
||||||
|
printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
|
||||||
|
exit 13
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'gitea runner lifecycle cleanup dry-run\n'
|
||||||
|
printf 'namespace: %s\n' "$namespace"
|
||||||
|
printf 'mode: %s\n\n' "$mode"
|
||||||
|
|
||||||
|
printf 'StatefulSet:\n'
|
||||||
|
kubectl -n "$namespace" get statefulset gitea-runner -o wide
|
||||||
|
|
||||||
|
printf '\nActive runner pods:\n'
|
||||||
|
kubectl -n "$namespace" get pods -l "$selector" -o wide
|
||||||
|
|
||||||
|
printf '\nRunner /data PVCs:\n'
|
||||||
|
kubectl -n "$namespace" get pvc -l "$selector" -o wide
|
||||||
|
|
||||||
|
printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
|
||||||
|
found_candidate=0
|
||||||
|
for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
pod="${pvc#data-}"
|
||||||
|
if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
|
||||||
|
found_candidate=1
|
||||||
|
printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$found_candidate" -eq 0 ]; then
|
||||||
|
printf 'none\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '\nGitea registration reconciliation:\n'
|
||||||
|
printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
|
||||||
|
printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-cleanup-dry-run
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
schedule: "17 3 * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
successfulJobsHistoryLimit: 3
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 3600
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
serviceAccountName: gitea-runner-lifecycle
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: cleanup-dry-run
|
||||||
|
image: bitnami/kubectl:1.30
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- /scripts/cleanup-dry-run.sh
|
||||||
|
env:
|
||||||
|
- name: RUNNER_NAMESPACE
|
||||||
|
value: gitea-runners
|
||||||
|
- name: CLEANUP_MODE
|
||||||
|
value: dry-run
|
||||||
|
volumeMounts:
|
||||||
|
- name: lifecycle-scripts
|
||||||
|
mountPath: /scripts
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: lifecycle-scripts
|
||||||
|
configMap:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
defaultMode: 0555
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
- service.yaml
|
||||||
|
- service-account.yaml
|
||||||
|
- runner-config.yaml
|
||||||
|
- statefulset.yaml
|
||||||
|
- cleanup-lifecycle.yaml
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-config
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
data:
|
||||||
|
config.yaml: |
|
||||||
|
log:
|
||||||
|
level: info
|
||||||
|
|
||||||
|
runner:
|
||||||
|
file: /data/.runner
|
||||||
|
capacity: 1
|
||||||
|
envs: {}
|
||||||
|
timeout: 3h
|
||||||
|
insecure: false
|
||||||
|
fetch_timeout: 5s
|
||||||
|
fetch_interval: 2s
|
||||||
|
labels:
|
||||||
|
- ubuntu-latest
|
||||||
|
# The nix label is intentionally disabled until the runner image has a
|
||||||
|
# concrete registry-reported digest; see ../runbook.md before deploy.
|
||||||
|
|
||||||
|
cache:
|
||||||
|
enabled: true
|
||||||
|
dir: /data/cache
|
||||||
|
|
||||||
|
container:
|
||||||
|
network: bridge
|
||||||
|
privileged: false
|
||||||
|
force_pull: true
|
||||||
|
valid_volumes: []
|
||||||
|
docker_host: unix:///runner-docker/docker.sock
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
rules: []
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: gitea-runner
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
ports:
|
||||||
|
- name: cache
|
||||||
|
port: 8088
|
||||||
|
targetPort: cache
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
serviceName: gitea-runner
|
||||||
|
replicas: 5
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
annotations:
|
||||||
|
hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
|
||||||
|
spec:
|
||||||
|
serviceAccountName: gitea-runner
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
securityContext:
|
||||||
|
fsGroup: 1000
|
||||||
|
containers:
|
||||||
|
- name: runner
|
||||||
|
image: gitea/act_runner:0.2.11
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: GITEA_INSTANCE_URL
|
||||||
|
value: https://gitea.hectic-lab.com
|
||||||
|
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
||||||
|
value: /runner-secrets/token
|
||||||
|
- name: CONFIG_FILE
|
||||||
|
value: /runner-config/config.yaml
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: unix:///runner-docker/docker.sock
|
||||||
|
ports:
|
||||||
|
- name: cache
|
||||||
|
containerPort: 8088
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- test -s /data/.runner && test -S /runner-docker/docker.sock
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 6
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- test -S /runner-docker/docker.sock
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 6
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
- name: config
|
||||||
|
mountPath: /runner-config
|
||||||
|
readOnly: true
|
||||||
|
- name: runner-token
|
||||||
|
mountPath: /runner-secrets
|
||||||
|
readOnly: true
|
||||||
|
- name: docker-socket
|
||||||
|
mountPath: /runner-docker
|
||||||
|
- name: docker
|
||||||
|
image: docker:27-dind
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- --host=unix:///runner-docker/docker.sock
|
||||||
|
- --storage-driver=overlay2
|
||||||
|
- --tls=false
|
||||||
|
env:
|
||||||
|
- name: DOCKER_TLS_CERTDIR
|
||||||
|
value: ""
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: unix:///runner-docker/docker.sock
|
||||||
|
securityContext:
|
||||||
|
# Privileged rootful DinD is intentionally scoped to this trusted
|
||||||
|
# internal runner pool; never expose it to untrusted fork/PR jobs.
|
||||||
|
privileged: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 4Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- docker
|
||||||
|
- info
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 6
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- docker
|
||||||
|
- info
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 6
|
||||||
|
volumeMounts:
|
||||||
|
- name: docker-socket
|
||||||
|
mountPath: /runner-docker
|
||||||
|
- name: docker-graph
|
||||||
|
mountPath: /var/lib/docker
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: gitea-runner-config
|
||||||
|
- name: runner-token
|
||||||
|
secret:
|
||||||
|
secretName: gitea-runner-token
|
||||||
|
items:
|
||||||
|
- key: token
|
||||||
|
path: token
|
||||||
|
defaultMode: 0400
|
||||||
|
- name: docker-socket
|
||||||
|
emptyDir: {}
|
||||||
|
- name: docker-graph
|
||||||
|
emptyDir: {}
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: hcloud-volumes
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# OpenTofu working directory and downloaded modules/providers.
|
||||||
|
.terraform/
|
||||||
|
.terraform.lock.hcl
|
||||||
|
|
||||||
|
# State must live in the S3 backend for production. Local state is allowed only
|
||||||
|
# for throwaway syntax checks with `tofu init -backend=false` and must not be
|
||||||
|
# committed.
|
||||||
|
terraform.tfstate
|
||||||
|
terraform.tfstate.*
|
||||||
|
*.tfstate
|
||||||
|
*.tfstate.*
|
||||||
|
crash.log
|
||||||
|
crash.*.log
|
||||||
|
|
||||||
|
# Plans can contain secrets or derived infrastructure data.
|
||||||
|
*.tfplan
|
||||||
|
*.plan
|
||||||
|
kubeconfig
|
||||||
|
kubeconfig.yaml
|
||||||
|
*_kubeconfig.yaml
|
||||||
|
|
||||||
|
# Variable files commonly carry credentials. Keep production inputs in SOPS or
|
||||||
|
# external environment/configuration, not in checked-in files.
|
||||||
|
*.tfvars
|
||||||
|
*.tfvars.json
|
||||||
|
override.tf
|
||||||
|
override.tf.json
|
||||||
|
*_override.tf
|
||||||
|
*_override.tf.json
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# Gitea runner OpenTofu backend contract
|
||||||
|
|
||||||
|
This directory defines the safe backend, provider contract, and kube-hetzner
|
||||||
|
cluster stack for the Gitea runner Kubernetes cluster.
|
||||||
|
|
||||||
|
## Required backend
|
||||||
|
|
||||||
|
Production state must use the OpenTofu S3 backend in `backend.tf`:
|
||||||
|
|
||||||
|
- bucket: `gitea-runner-hectic-lab`
|
||||||
|
- key: `gitea-runners/kube-hetzner/terraform.tfstate`
|
||||||
|
- region: `fsn1`, aligned with the target Hetzner location
|
||||||
|
- encryption: `encrypt = true`
|
||||||
|
- locking: `use_lockfile = true` where the selected S3-compatible endpoint
|
||||||
|
supports it
|
||||||
|
|
||||||
|
Before any production `tofu init`, verify the S3-compatible endpoint, credential
|
||||||
|
source, bucket versioning, encryption behavior, and lockfile support for the
|
||||||
|
chosen object-storage provider. Keep backend authentication externalized through
|
||||||
|
environment variables, AWS-compatible shared config, or the production secret
|
||||||
|
injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
|
||||||
|
tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
|
||||||
|
OpenTofu files.
|
||||||
|
|
||||||
|
## Local state safety
|
||||||
|
|
||||||
|
Production local state is forbidden. Only syntax-only validation/prototyping may
|
||||||
|
use local state, and it must use backend-disabled initialization:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init -backend=false
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
```
|
||||||
|
|
||||||
|
Fail the run if production local state appears:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test ! -e infra/gitea-runners/opentofu/terraform.tfstate
|
||||||
|
test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
|
||||||
|
grep -R 'backend "s3"' infra/gitea-runners/opentofu
|
||||||
|
```
|
||||||
|
|
||||||
|
The `.gitignore` in this directory blocks local state, plans, downloaded
|
||||||
|
providers/modules, and variable files from being committed. Treat any local
|
||||||
|
state file as disposable validation residue, never as production state.
|
||||||
|
|
||||||
|
## Provider and module pins
|
||||||
|
|
||||||
|
`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
|
||||||
|
`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
|
||||||
|
observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
|
||||||
|
module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
|
||||||
|
as locals so Task 5 can wire the module without re-opening the version contract.
|
||||||
|
|
||||||
|
`providers.tf` leaves the `hcloud` provider empty so authentication comes from
|
||||||
|
the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
|
||||||
|
Do not set token values in `.tf` or `.tfvars` files.
|
||||||
|
|
||||||
|
## Cluster shape
|
||||||
|
|
||||||
|
The default cluster is deliberately fixed-size:
|
||||||
|
|
||||||
|
- cluster name: `gitea-runners`
|
||||||
|
- Hetzner location: `fsn1`
|
||||||
|
- private network region: `eu-central`
|
||||||
|
- control plane: one `cpx21` node in pool `control-plane`
|
||||||
|
- workers: three `cpx31` nodes in pool `runner-workers`
|
||||||
|
- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
|
||||||
|
- Longhorn: disabled
|
||||||
|
- autoscaling/KEDA: not enabled in this stack
|
||||||
|
|
||||||
|
The three default workers are sized for the initial five trusted privileged DinD
|
||||||
|
jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
|
||||||
|
`worker_count` to `5` or increase `worker_server_type`, then run a fresh
|
||||||
|
`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
|
||||||
|
|
||||||
|
Required inputs must come from environment or secret injection, for example
|
||||||
|
`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
|
||||||
|
Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
|
||||||
|
kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
|
||||||
|
enabled; this path is ignored as operational secret material.
|
||||||
|
|
||||||
|
## Known state caveat
|
||||||
|
|
||||||
|
kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
|
||||||
|
internal `kube_system_secrets` handling. This task does not claim that risk is
|
||||||
|
solved. Task 5 must verify the generated plan and state before production apply
|
||||||
|
and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
|
||||||
|
keys, and decrypted secrets are absent from committed files and unsafe state
|
||||||
|
evidence.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
terraform {
|
||||||
|
backend "s3" {
|
||||||
|
bucket = "gitea-runner-hectic-lab"
|
||||||
|
key = "gitea-runners/kube-hetzner/terraform.tfstate"
|
||||||
|
region = "fsn1"
|
||||||
|
encrypt = true
|
||||||
|
use_lockfile = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
check "remote_state_contract" {
|
||||||
|
assert {
|
||||||
|
condition = local.production_remote_state
|
||||||
|
error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
locals {
|
||||||
|
default_storage_class = "hcloud-volumes"
|
||||||
|
|
||||||
|
control_plane_nodepools = [
|
||||||
|
{
|
||||||
|
name = "control-plane"
|
||||||
|
server_type = var.control_plane_server_type
|
||||||
|
location = var.hetzner_location
|
||||||
|
labels = []
|
||||||
|
taints = []
|
||||||
|
count = 1
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
agent_nodepools = [
|
||||||
|
{
|
||||||
|
name = "runner-workers"
|
||||||
|
server_type = var.worker_server_type
|
||||||
|
location = var.hetzner_location
|
||||||
|
labels = ["node-role.hectic-lab/gitea-runner=true"]
|
||||||
|
taints = []
|
||||||
|
count = var.worker_count
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
module "kube_hetzner" {
|
||||||
|
source = "kube-hetzner/kube-hetzner/hcloud"
|
||||||
|
version = "2.19.3"
|
||||||
|
|
||||||
|
providers = {
|
||||||
|
hcloud = hcloud
|
||||||
|
}
|
||||||
|
|
||||||
|
hcloud_token = var.hcloud_token
|
||||||
|
ssh_public_key = var.ssh_public_key
|
||||||
|
ssh_private_key = var.ssh_private_key
|
||||||
|
|
||||||
|
cluster_name = var.cluster_name
|
||||||
|
base_domain = var.base_domain
|
||||||
|
|
||||||
|
# kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
|
||||||
|
# expose that expected path without outputting kubeconfig private key material.
|
||||||
|
create_kubeconfig = true
|
||||||
|
|
||||||
|
network_region = var.network_region
|
||||||
|
load_balancer_location = var.hetzner_location
|
||||||
|
control_plane_nodepools = local.control_plane_nodepools
|
||||||
|
agent_nodepools = local.agent_nodepools
|
||||||
|
|
||||||
|
# Hetzner CSI is the required StorageClass provider for runner PVCs.
|
||||||
|
disable_hetzner_csi = false
|
||||||
|
|
||||||
|
# Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
|
||||||
|
enable_longhorn = false
|
||||||
|
|
||||||
|
# Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
|
||||||
|
# raise worker_count to 5 or increase worker_server_type after validating pod
|
||||||
|
# CPU, memory, and ephemeral-storage pressure in Task 11.
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
output "kubeconfig_path" {
|
||||||
|
description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
|
||||||
|
value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
|
||||||
|
}
|
||||||
|
|
||||||
|
output "cluster_name" {
|
||||||
|
description = "kube-hetzner cluster name."
|
||||||
|
value = var.cluster_name
|
||||||
|
}
|
||||||
|
|
||||||
|
output "node_pool_names" {
|
||||||
|
description = "Control-plane and worker node pool names used by this stack."
|
||||||
|
value = {
|
||||||
|
control_plane = [for pool in local.control_plane_nodepools : pool.name]
|
||||||
|
workers = [for pool in local.agent_nodepools : pool.name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output "default_storage_class" {
|
||||||
|
description = "Default Hetzner CSI StorageClass expected for runner PVCs."
|
||||||
|
value = local.default_storage_class
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
provider "hcloud" {}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
variable "hcloud_token" {
|
||||||
|
description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_public_key" {
|
||||||
|
description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_private_key" {
|
||||||
|
description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "cluster_name" {
|
||||||
|
description = "Name for the kube-hetzner runner cluster."
|
||||||
|
type = string
|
||||||
|
default = "gitea-runners"
|
||||||
|
|
||||||
|
validation {
|
||||||
|
condition = can(regex("^[a-z0-9-]+$", var.cluster_name))
|
||||||
|
error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "hetzner_location" {
|
||||||
|
description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
|
||||||
|
type = string
|
||||||
|
default = "fsn1"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "network_region" {
|
||||||
|
description = "Hetzner private network region. eu-central covers fsn1."
|
||||||
|
type = string
|
||||||
|
default = "eu-central"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "control_plane_server_type" {
|
||||||
|
description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
|
||||||
|
type = string
|
||||||
|
default = "cpx21"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "worker_server_type" {
|
||||||
|
description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
|
||||||
|
type = string
|
||||||
|
default = "cpx31"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "worker_count" {
|
||||||
|
description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
|
||||||
|
type = number
|
||||||
|
default = 3
|
||||||
|
|
||||||
|
validation {
|
||||||
|
condition = var.worker_count >= 1
|
||||||
|
error_message = "worker_count must be at least 1."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "kubeconfig_path" {
|
||||||
|
description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
|
||||||
|
type = string
|
||||||
|
default = null
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "base_domain" {
|
||||||
|
description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
|
||||||
|
type = string
|
||||||
|
default = ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
terraform {
|
||||||
|
required_version = ">= 1.10.1"
|
||||||
|
|
||||||
|
required_providers {
|
||||||
|
hcloud = {
|
||||||
|
source = "hetznercloud/hcloud"
|
||||||
|
version = "1.60.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud"
|
||||||
|
kube_hetzner_module_version = "2.19.3"
|
||||||
|
hcloud_provider_minimum = ">= 1.59.0"
|
||||||
|
production_remote_state = true
|
||||||
|
}
|
||||||
@@ -0,0 +1,503 @@
|
|||||||
|
# Gitea Runner Infrastructure Runbook
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This directory is the repo-owned boundary for the first Gitea Actions runner
|
||||||
|
pool. Task 1 only establishes the scaffold and immutable decision contract;
|
||||||
|
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
|
||||||
|
and a Nix-capable runner image under the existing subdirectories.
|
||||||
|
|
||||||
|
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
|
||||||
|
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
|
||||||
|
workflow job containers are ephemeral, while each runner pod keeps its runner
|
||||||
|
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||||
|
|
||||||
|
## Immutable decisions
|
||||||
|
|
||||||
|
- Infrastructure is managed with OpenTofu command examples only, using the
|
||||||
|
`tofu` CLI.
|
||||||
|
- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
|
||||||
|
- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
|
||||||
|
- Runner implementation is the non-Enterprise `gitea/runner`.
|
||||||
|
- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
|
||||||
|
- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
|
||||||
|
file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
|
||||||
|
environment variables are not the contract.
|
||||||
|
- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
|
||||||
|
`/data`, including `/data/.runner`.
|
||||||
|
- Container builds run through privileged rootful DinD inside trusted runner
|
||||||
|
pods; host Docker socket mounting is not an implementation path.
|
||||||
|
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
|
||||||
|
the Nix-capable image has been pushed and a concrete registry-reported digest
|
||||||
|
is added to the runner ConfigMap.
|
||||||
|
- First scope is trusted internal workflows only, with no untrusted fork or PR
|
||||||
|
workflow support.
|
||||||
|
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
|
||||||
|
|
||||||
|
## Lifecycle boundaries
|
||||||
|
|
||||||
|
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
|
||||||
|
contract, Hetzner provider configuration, and kube-hetzner module wiring.
|
||||||
|
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
|
||||||
|
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
|
||||||
|
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
|
||||||
|
handoff; package or flake output changes are outside Task 1.
|
||||||
|
- `infra/gitea-runners/runbook.md`: this contract plus later operational
|
||||||
|
commands, rollback notes, and acceptance evidence references.
|
||||||
|
|
||||||
|
## Guardrails
|
||||||
|
|
||||||
|
- Enterprise ARC/actions-runner-controller are rejected alternatives and must
|
||||||
|
not be implemented here. Do not add ARC custom resources, controller install
|
||||||
|
instructions, or GitHub Actions ARC assumptions.
|
||||||
|
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
|
||||||
|
acceptable for trusted internal jobs.
|
||||||
|
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
|
||||||
|
runner pool.
|
||||||
|
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
|
||||||
|
S3 credentials, decrypted SOPS files, or SOPS age keys.
|
||||||
|
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
|
||||||
|
runner token; Kubernetes receives the token as a mounted file secret instead.
|
||||||
|
- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
|
||||||
|
jobs must reach the public HTTPS service.
|
||||||
|
|
||||||
|
## Initial acceptance commands
|
||||||
|
|
||||||
|
Run from the repository root:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
|
||||||
|
test -f infra/gitea-runners/runbook.md
|
||||||
|
grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
|
||||||
|
grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
|
||||||
|
grep -R "[t]erraform " infra/gitea-runners || true
|
||||||
|
grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected outcomes: the directory and file checks exit 0; the architecture-term
|
||||||
|
grep shows this contract; ARC references appear only in the rejected-alternative
|
||||||
|
guardrail above; there are no forbidden CLI command examples and no unresolved
|
||||||
|
decision placeholders.
|
||||||
|
|
||||||
|
## Downstream placeholders
|
||||||
|
|
||||||
|
- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
|
||||||
|
- Task 3: add SOPS secret contract and runtime token delivery details.
|
||||||
|
- Task 4: define or package the Nix-capable runner image for the `nix` label.
|
||||||
|
- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
|
||||||
|
and document cleanup, rollback, and scaling operations.
|
||||||
|
|
||||||
|
## Runner lifecycle cleanup
|
||||||
|
|
||||||
|
All lifecycle commands are scoped to the runner namespace:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
|
||||||
|
runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
|
||||||
|
does not delete pods, PVCs, Docker data, or Gitea runner registrations.
|
||||||
|
|
||||||
|
Run the same inventory on demand without waiting for the schedule:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||||
|
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||||
|
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
|
||||||
|
manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
|
||||||
|
operator to remove finished jobs manually.
|
||||||
|
|
||||||
|
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||||
|
pod loses `/data/.runner`. That runner identity must then be deregistered from
|
||||||
|
Gitea or the replacement pod must be allowed to re-register intentionally with
|
||||||
|
the current organization runner token. Do not delete an active runner PVC as a
|
||||||
|
normal cleanup step.
|
||||||
|
|
||||||
|
Non-UI Gitea registration reconciliation uses the Gitea API with a separate
|
||||||
|
admin token. Store that token outside this repository and pass it as a file; do
|
||||||
|
not print it:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||||
|
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||||
|
--restart=Never \
|
||||||
|
--image=curlimages/curl:8.10.1 \
|
||||||
|
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
|
||||||
|
has completed and its logs have been collected. Do not keep this admin token in
|
||||||
|
the runner namespace longer than the reconciliation window.
|
||||||
|
|
||||||
|
Only remove a stale Gitea runner registration after the corresponding pod/PVC
|
||||||
|
was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
|
||||||
|
Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
|
||||||
|
cleanup is a fallback, not the only path. Record the removed runner name and the
|
||||||
|
Kubernetes PVC/pod deletion that made it stale.
|
||||||
|
|
||||||
|
After the dry-run list identifies a stale registration and the PVC/pod deletion
|
||||||
|
has been recorded, remove that exact Gitea runner by id through the API:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||||
|
umask 077
|
||||||
|
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||||
|
trap 'rm -f "$curl_config"' EXIT
|
||||||
|
{
|
||||||
|
printf 'request = "DELETE"\n'
|
||||||
|
printf 'header = "Authorization: token '
|
||||||
|
cat /secure/path/gitea-admin-token
|
||||||
|
printf '"\n'
|
||||||
|
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||||
|
} > "$curl_config"
|
||||||
|
curl -fsS --config "$curl_config"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run the delete command for a runner that still has an active
|
||||||
|
`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
|
||||||
|
being intentionally reset for re-registration.
|
||||||
|
|
||||||
|
## Docker-in-Docker storage cleanup
|
||||||
|
|
||||||
|
Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
|
||||||
|
pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
|
||||||
|
list disk usage before pruning, and run the command only against the `docker`
|
||||||
|
container in runner pods in `gitea-runners`. Because this storage is pod-local,
|
||||||
|
loop over pods for pool-wide cleanup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
For one pod, replace the StatefulSet target with the pod name:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
|
||||||
|
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run host-level Docker cleanup commands and do not mount or prune a host
|
||||||
|
Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
|
||||||
|
Kubernetes; the `/data` PVC remains and still controls runner identity.
|
||||||
|
|
||||||
|
## Token rotation
|
||||||
|
|
||||||
|
Rotate the Gitea organization runner token without printing decrypted values:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sops sus/gitea-runners.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
kubectl -n gitea-runners rollout restart statefulset/gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||||
|
```
|
||||||
|
|
||||||
|
The `rollout restart` command above is the controlled restart path for this
|
||||||
|
StatefulSet. Observe the rollout and each ordinal until all replacement pods are
|
||||||
|
Ready; do not delete runner pods directly as part of normal token rotation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
Verification must confirm the token file mount remains present while the token
|
||||||
|
value never appears in logs or evidence:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deploy and status
|
||||||
|
|
||||||
|
These commands are executable only when the external inputs are available:
|
||||||
|
|
||||||
|
- `TF_VAR_hcloud_token`
|
||||||
|
- `TF_VAR_ssh_public_key`
|
||||||
|
- `TF_VAR_ssh_private_key`
|
||||||
|
- S3 backend credentials and endpoint access
|
||||||
|
- a matching SOPS age identity for `sus/gitea-runners.yaml`
|
||||||
|
- `kubectl` access to the target cluster
|
||||||
|
- a concrete digest for the pushed Nix-capable runner image, if enabling the
|
||||||
|
`nix` label
|
||||||
|
|
||||||
|
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
|
||||||
|
stale kubeconfig files.
|
||||||
|
|
||||||
|
Before production Kubernetes apply or rollout, satisfy both manifest gates:
|
||||||
|
|
||||||
|
1. Create or update the `gitea-runner-token` Secret from SOPS. The active
|
||||||
|
Kustomize overlay intentionally does not include a placeholder Secret, but
|
||||||
|
the StatefulSet still mounts `secretName: gitea-runner-token` as
|
||||||
|
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
|
||||||
|
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
|
||||||
|
image has been pushed successfully. Enable the `nix` label only by adding a
|
||||||
|
digest-pinned `docker://` mapping with the exact registry-reported sha256
|
||||||
|
digest from that push.
|
||||||
|
|
||||||
|
Use the same SOPS materialization pattern as token rotation before applying the
|
||||||
|
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
|
||||||
|
target namespace; the full overlay remains gated on the Secret and digest
|
||||||
|
decisions:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
|
||||||
|
above succeeds. Do not claim or enable the `nix` runner label until the image
|
||||||
|
publication step has produced the concrete digest.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||||
|
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||||
|
kubectl config current-context
|
||||||
|
kubectl get nodes -o wide
|
||||||
|
kubectl get sc
|
||||||
|
kubectl apply -k infra/gitea-runners/k8s
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected status after deploy:
|
||||||
|
|
||||||
|
- `kubectl config current-context` names the runner cluster context.
|
||||||
|
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
|
||||||
|
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
|
||||||
|
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
|
||||||
|
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
|
||||||
|
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
|
||||||
|
|
||||||
|
## Scale 5 to 10 to 5
|
||||||
|
|
||||||
|
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
|
||||||
|
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
|
||||||
|
# Run the concurrent smoke workflows now.
|
||||||
|
|
||||||
|
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
After scaling back down, inspect the cleanup dry-run and deregister any stale
|
||||||
|
runner registrations only for pods or PVCs that were intentionally removed.
|
||||||
|
|
||||||
|
## Cleanup and stale runner deregistration
|
||||||
|
|
||||||
|
Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
|
||||||
|
PVC candidates whose pod is gone. It must not delete active resources.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||||
|
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||||
|
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Pool-wide DinD storage checks and cleanup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||||
|
pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
|
||||||
|
replacement pod re-register intentionally with the current organization token.
|
||||||
|
Never delete an active runner PVC as routine cleanup.
|
||||||
|
|
||||||
|
Non-UI Gitea registration reconciliation uses an admin token stored outside this
|
||||||
|
repository:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||||
|
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||||
|
--restart=Never \
|
||||||
|
--image=curlimages/curl:8.10.1 \
|
||||||
|
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
|
||||||
|
pod has completed and its logs have been collected.
|
||||||
|
|
||||||
|
After the dry-run list identifies a stale registration and the PVC or pod
|
||||||
|
deletion has been recorded, remove that exact Gitea runner by id through the
|
||||||
|
API:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||||
|
umask 077
|
||||||
|
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||||
|
trap 'rm -f "$curl_config"' EXIT
|
||||||
|
{
|
||||||
|
printf 'request = "DELETE"\n'
|
||||||
|
printf 'header = "Authorization: token '
|
||||||
|
cat /secure/path/gitea-admin-token
|
||||||
|
printf '"\n'
|
||||||
|
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||||
|
} > "$curl_config"
|
||||||
|
curl -fsS --config "$curl_config"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run the delete command for a runner that still has an active
|
||||||
|
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
|
||||||
|
being intentionally reset for re-registration.
|
||||||
|
|
||||||
|
## Application rollback
|
||||||
|
|
||||||
|
Rollback the app layer only. Do not use this section to destroy the cluster.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners rollout history statefulset/gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
```
|
||||||
|
|
||||||
|
If a manifest rollback is needed, reapply the repo overlay after checking out the
|
||||||
|
known-good revision, then re-run the rollout checks:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
```
|
||||||
|
|
||||||
|
## Full cluster teardown
|
||||||
|
|
||||||
|
This destroys Hetzner resources owned by the kube-hetzner stack, including the
|
||||||
|
`gitea-runners` cluster nodes, the `control-plane` node pool, the
|
||||||
|
`runner-workers` node pool, the cluster network, load balancer resources,
|
||||||
|
firewall objects, and any attached Hetzner CSI volumes still managed by the
|
||||||
|
stack. Do not run teardown unless the destruction is intentional.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
|
||||||
|
```
|
||||||
|
|
||||||
|
## Partial OpenTofu apply recovery
|
||||||
|
|
||||||
|
If `tofu apply` fails after creating some resources, do not destroy blindly.
|
||||||
|
First reconcile state and inspect what the stack thinks exists:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu state list
|
||||||
|
```
|
||||||
|
|
||||||
|
Then rerun the normal plan path. Use `-target` only as a last resort when a
|
||||||
|
single resource is stuck and the drift is understood.
|
||||||
|
|
||||||
|
## S3 backend recovery
|
||||||
|
|
||||||
|
If backend init or state access fails, first verify the bucket and versioning
|
||||||
|
outside OpenTofu, then reconfigure the backend:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
|
||||||
|
nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan
|
||||||
|
```
|
||||||
|
|
||||||
|
If the backend reports a stale lock, confirm no `tofu` process is active, then
|
||||||
|
use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
|
||||||
|
unlock a live plan or apply.
|
||||||
|
|
||||||
|
## Gitea outage troubleshooting
|
||||||
|
|
||||||
|
Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
|
||||||
|
containers.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
```
|
||||||
|
|
||||||
|
If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
|
||||||
|
`/data/.runner` just because the service is unavailable. Once Gitea returns,
|
||||||
|
repeat the token rotation or re-registration path if a pod restarted while the
|
||||||
|
service was unavailable and lost its runner identity.
|
||||||
|
|
||||||
|
## Release checklist
|
||||||
|
|
||||||
|
Do not release unless the following evidence files exist and are readable:
|
||||||
|
|
||||||
|
- `.sisyphus/evidence/task-5-cluster-plan.txt`
|
||||||
|
- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
|
||||||
|
- `.sisyphus/evidence/task-9-deploy.txt`
|
||||||
|
- `.sisyphus/evidence/task-9-secret-mount.txt`
|
||||||
|
- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
|
||||||
|
- `.sisyphus/evidence/task-10-nix-workflow.txt`
|
||||||
|
- `.sisyphus/evidence/task-11-scale.txt`
|
||||||
|
- `.sisyphus/evidence/task-11-restart-cleanup.txt`
|
||||||
|
|
||||||
|
If any evidence file is missing, stop and collect it before treating the runbook
|
||||||
|
as complete.
|
||||||
@@ -53,9 +53,11 @@ dockerTools.buildLayeredImageWithNixDb {
|
|||||||
chmod 1777 tmp
|
chmod 1777 tmp
|
||||||
|
|
||||||
cat > etc/nix/nix.conf <<'EOF'
|
cat > etc/nix/nix.conf <<'EOF'
|
||||||
|
accept-flake-config = true
|
||||||
experimental-features = nix-command flakes
|
experimental-features = nix-command flakes
|
||||||
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||||
|
trusted-users = root
|
||||||
sandbox = false
|
sandbox = false
|
||||||
EOF
|
EOF
|
||||||
'';
|
'';
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
gitea:
|
||||||
|
hectic-lab:
|
||||||
|
org-runner-registration-token: ENC[AES256_GCM,data:ouupBDavLapNezKtcIViqfhsiQJTXwOL3eE9lLDutIvxk+oxOxUmbO4=,iv:Bck5ZYbDlYnBADSLk4lLtskzX3bqFCVUzj6AbWWJXBU=,tag:8y8nHDELNvlDEkrw0bwXKw==,type:str]
|
||||||
|
hetzner:
|
||||||
|
hcloud-token: ENC[AES256_GCM,data:uvqCdk+0S3HhSeo/NlGj42ii6Q9aCKq6r0XLuoelBJA=,iv:0sDmmulU4IWIqvhS9n2BTGUutuvZq+8kqW79nBaShBk=,tag:qnQxz5j8sTP9JA5hNFWliw==,type:str]
|
||||||
|
s3:
|
||||||
|
access-key-id: ENC[AES256_GCM,data:fzYW2dI2q43lx7Mb1vMzDAGQeps65ro/0XroLA==,iv:bCmlH4J0Jo0ADSgQ+XxlB4Pjk+1pkmILzYbpfHGyaEs=,tag:m5YtDNOj8u7F1OVtM9NLvQ==,type:str]
|
||||||
|
secret-access-key: ENC[AES256_GCM,data:wxODth7RJNQsmbqr3stAclVAsoeU2BbY94GfkSyORzs=,iv:9QUOyK/2Kc2VuIylsT5d7WOQIiUdwpuzTNvwTW02JBg=,tag:7oY4p1dMzB1A3AeCYYmswA==,type:str]
|
||||||
|
sops:
|
||||||
|
age:
|
||||||
|
- recipient: age1x04u7ftjgx8de2gq596e7frauze764cmn7jjwqnx8szthvfft5qq0tezx6
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNjZxVlMwUkpuU3lOZkxZ
|
||||||
|
SlQwRk5vQmRQdWprS20zSmJncjBpMExjVkdzCk9RV2lrTjE4UXZPMGtXK0NJMEVR
|
||||||
|
cDhDOXNpL1NYNHpKais1VEs0Si9EOG8KLS0tIFo2QmpFMEtYQVBVYjdpOEorblRr
|
||||||
|
L3l6MjNZdHZHblZJejdmRXpVUWIxb0UK4xP/kIxCge5t0XGv70BUt4oLUGHB/6Pj
|
||||||
|
2bX+gUq3KMzgEJahOgN67EJBJj7Ixs9B+MpBZkmTE9WddvyHyXffwQ==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1r25zdeqq8nac6dgca9en28r57ffyz9u9d8z5yc25gc8xqz747vaqmdtk0h
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTkpzdUo5V2FzWHlWbkFi
|
||||||
|
VmhtNWhoR1pCd2RRWWJxYlV4YVRvSThmSXlFCmtuVDlFd1lJa21xaDFTQ3JhazZB
|
||||||
|
RkM1ZkRyS0VGK1VUajFXeEozZEZYMmMKLS0tIHRwT2JiNGJmYjdvUGpRT3o2cTNC
|
||||||
|
UTFIbFZwYXo0T0REQkRNT3ZLS0VjRjgKCompPQJCfhxF/fNYT909szq5KDVDkzzB
|
||||||
|
9CKFoGVRYk+5cbBk1zprGuqoBCWEJ85MYKjueEJKowxlCb+8/rr/Ug==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1vv46vn4hsn2lg6jy834cpu40c3mvqklldcm3hjtynrhwtpmlpc8szruz4v
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPMzM3bExWZlRVRHl3eHg5
|
||||||
|
ai9SUTU3YjRodm90T2poc2dJRTkzcWJic1RRCmJMVytUK0pQWk5KVTJNT0RSZUUz
|
||||||
|
TS8xRHNLQjVZM2RGY0JjTXJrMjFrSlUKLS0tIE9nd29kNXRhK2xKVlEvR29HbFht
|
||||||
|
ZU1jOGprOXhZTC9HZ0RUZFpZeFV6eGsKhw+TXpiEd7dYPG84xjr3rV/dPm0Nilgl
|
||||||
|
509h/L3qqp7pJUOjZWidMB63dr1/DlTxjRIKubBy0eqidCcnkDnDfQ==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age13h8twnwvgxn04l5ywtru89a6psw5d0uckr2eghxsjp88a5augvsstq5ard
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFbjF2ZkpKUlZFRE9CUEw0
|
||||||
|
Y3kvRzM0blNHdkFEMlFxdHdNQjY1cFdKWjA4CkNPdHZnM3NqQXZId05UQWFPQ1B1
|
||||||
|
d0RrM2lBb3JNMlBNV2VCTFhPS3BHMEUKLS0tIHpNMEt3WnJlMWJVWHIzbjdVdkFL
|
||||||
|
WklFVDlBOWY4Kzd6TzlOKzBPNmtobHMKw0zroE9HR0yJPebYPpCJ0ubB0WSPagQ9
|
||||||
|
u01s9TGMYtxYsD/WOCo/gJSGbilixl4RbmQzpxfzklco2ht73M1q3Q==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1jxntjca8q2vxvf2jaal4xyvm2ae6sh62fhv897694kuzawfrk5asj00zdt
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZTQ4dk9OdlVWaVVGdm12
|
||||||
|
QXVVVlpUaFFOOTByQVozWVJYdXdBRitsZHlFCkxyWlhqR0NaKzd0Y3lJcFg3eGZB
|
||||||
|
ejBMbnVvZXFvUkxsRitvQ0N3Vnh6dWsKLS0tIHM1aG5zTGE3QVZSdEdNbGtvekxB
|
||||||
|
UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P
|
||||||
|
MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
lastmodified: "2026-06-07T22:55:43Z"
|
||||||
|
mac: ENC[AES256_GCM,data:+FNn3iLTgWF5470FCDus3uBpSir3E0nyajtwAjTj4yoVBaGk1QcFqDxA+S9wD85Ch050uHyeiR+JlApeeNltZtnidPIStQdj5tdP2UP+b2v8se4UzQyPCWTPN0wRIZ9JMubRhErZlCGovJlr5UHYdg/+gFfICPOthmFeCqEf5tM=,iv:Vq64SW0srhiappVknbdTtqichtsUDB0LeBlns43dO4Q=,tag:pVA0K2i3pP2Cpo1VSOf8dQ==,type:str]
|
||||||
|
unencrypted_suffix: _unencrypted
|
||||||
|
version: 3.10.2
|
||||||
Reference in New Issue
Block a user