feat: gitea runners infra, but it is so expancive

devshell/gitea-runners.nix

@@ -5,13 +5,202 @@
     exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
   '';
 
+  packer = pkgs.stdenvNoCC.mkDerivation {
+    pname = "packer";
+    version = "1.16.0";
+    src = pkgs.fetchurl {
+      url = "https://releases.hashicorp.com/packer/1.16.0/packer_1.16.0_linux_amd64.zip";
+      hash = "sha256-XtzRSrWbU1BAxRLb7Nbsnvl2oACwc8Gdk+TEMclIWB4=";
+    };
+    nativeBuildInputs = [ pkgs.unzip ];
+    dontUnpack = true;
+    dontConfigure = true;
+    dontBuild = true;
+    installPhase = ''
+      install -d $out/bin
+      unzip -p $src packer > $out/bin/packer
+      chmod 755 $out/bin/packer
+    '';
+  };
+
+  buildMicroosSnapshots = pkgs.writeShellScriptBin "gitea-runners-build-microos-snapshots" ''
+    set -eu
+
+    : "''${HCLOUD_TOKEN:?HCLOUD_TOKEN is not set}"
+
+    architecture="''${1:-both}"
+    x86_location="''${GITEA_RUNNERS_X86_LOCATION:-nbg1}"
+    x86_server_type="''${GITEA_RUNNERS_X86_SERVER_TYPE:-cx23}"
+    case "$architecture" in
+      x86|arm|both) ;;
+      *)
+        printf 'usage: gitea-runners-build-microos-snapshots [x86|arm|both]\n' >&2
+        exit 2
+        ;;
+    esac
+
+    workdir="$(mktemp -d)"
+    trap 'rm -rf "$workdir"' EXIT HUP INT TERM
+    ssh_private_key_file="$workdir/packer-ssh-key"
+    ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file"
+    image_public_key_file="''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_ed25519.pub}"
+    test -r "$image_public_key_file"
+    packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")"
+    image_public_key_b64="$(base64 -w0 "$image_public_key_file")"
+
+    curl -fsSL \
+      https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh \
+      -o "$workdir/create.sh"
+    chmod +x "$workdir/create.sh"
+
+    (
+      cd "$workdir"
+      folder_name="runner-images" \
+      folder_path="$workdir" \
+      create_snapshots=none \
+      HCLOUD_TOKEN="$HCLOUD_TOKEN" \
+      "$workdir/create.sh"
+    )
+
+    packer_dir="$workdir/runner-images/packer"
+    "$packer_dir/scripts/install-verified-packer-plugin-hcloud.sh"
+
+    # Packer may try to remove its remote script after the image-writing reboot,
+    # while SSH is already unavailable. Keep that cleanup from aborting builds.
+    sed -i \
+      '/inline[[:space:]]*=[[:space:]]*\[local\.write_x86_image\]/a\    skip_clean       = true' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    sed -i \
+      '/inline[[:space:]]*=[[:space:]]*\[local\.write_arm_image\]/a\    skip_clean       = true' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    sed -i \
+      '/inline[[:space:]]*=[[:space:]]*\[local\.install_packages\]/a\    start_retry_timeout = "15m"' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    sed -i \
+      '/ssh_username[[:space:]]*=[[:space:]]*"root"/a\  temporary_key_pair_type = "ed25519"' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    awk \
+      -v ssh_private_key_file="$ssh_private_key_file" \
+      '/token[[:space:]]*=[[:space:]]*var[.]hcloud_token/ {
+        print
+        print "  ssh_private_key_file = \"" ssh_private_key_file "\""
+        next
+      }
+      { print }' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
+      > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
+    mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+
+    cat > "$workdir/image-key-injection.txt" <<'EOF'
+  partprobe /dev/sda || true
+  udevadm settle
+  root_device=""
+  for candidate in /dev/sda[0-9]*; do
+    if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then
+      root_device="$candidate"
+      break
+    fi
+  done
+  test -n "$root_device"
+  mount -o subvol=@ "$root_device" /mnt
+  install -d -m 0700 /mnt/root/.ssh
+  printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/authorized_keys
+  printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/authorized_keys
+  chmod 0600 /mnt/root/.ssh/authorized_keys
+  sync
+  umount /mnt
+EOF
+    sed -i "s|__PACKER_PUBLIC_KEY_B64__|$packer_public_key_b64|" \
+      "$workdir/image-key-injection.txt"
+    sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \
+      "$workdir/image-key-injection.txt"
+    awk -v inject_file="$workdir/image-key-injection.txt" \
+      '/done[.] Rebooting/ {
+        while ((getline line < inject_file) > 0) print line
+        close(inject_file)
+      }
+      { print }' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
+      > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
+    mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+
+    cat > "$workdir/cloud-init-cleanup.txt" <<'EOF'
+  cloud-init clean --logs --machine-id --seed --configs all || true
+  rm -rf /run/cloud-init/* /var/lib/cloud/*
+EOF
+    awk -v cleanup_file="$workdir/cloud-init-cleanup.txt" \
+      '/# Cleanup some logs/ {
+        while ((getline line < cleanup_file) > 0) print "- [sh, -c, \"" line "\"]"
+        close(cleanup_file)
+      }
+      { print }' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
+      > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
+    mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+
+    x86_base='https://download.opensuse.org/tumbleweed/appliances'
+    x86_file='openSUSE-MicroOS.x86_64-ContainerHost-OpenStack-Cloud.qcow2'
+    x86_url="$x86_base/$x86_file"
+
+    arm_base='https://download.opensuse.org/ports/aarch64/tumbleweed/appliances'
+    arm_file='openSUSE-MicroOS.aarch64-ContainerHost-OpenStack-Cloud.qcow2'
+    arm_url="$arm_base/$arm_file"
+
+    x86_sha="$(curl -fsSL "$x86_url.sha256" | awk '{print $1; exit}')"
+    arm_sha="$(curl -fsSL "$arm_url.sha256" | awk '{print $1; exit}')"
+
+    test -n "$x86_sha"
+    test -n "$arm_sha"
+
+    printf 'x86 digest: %s\n' "$x86_sha"
+    printf 'arm digest: %s\n' "$arm_sha"
+
+    (
+      cd "$packer_dir"
+      packer init hcloud-microos-snapshots.pkr.hcl
+      case "$architecture" in
+        x86)
+          packer build \
+            -only=hcloud.microos-x86-snapshot \
+            -var 'selinux_package_to_install=k3s' \
+            -var "x86_location=$x86_location" \
+            -var "x86_server_type=$x86_server_type" \
+            -var "opensuse_microos_x86_expected_sha256=$x86_sha" \
+            -var "opensuse_microos_arm_expected_sha256=$arm_sha" \
+            hcloud-microos-snapshots.pkr.hcl
+          ;;
+        arm)
+          packer build \
+            -only=hcloud.microos-arm-snapshot \
+            -var 'selinux_package_to_install=k3s' \
+            -var "opensuse_microos_x86_expected_sha256=$x86_sha" \
+            -var "opensuse_microos_arm_expected_sha256=$arm_sha" \
+            hcloud-microos-snapshots.pkr.hcl
+          ;;
+        both)
+          packer build \
+            -var 'selinux_package_to_install=k3s' \
+            -var "x86_location=$x86_location" \
+            -var "x86_server_type=$x86_server_type" \
+            -var "opensuse_microos_x86_expected_sha256=$x86_sha" \
+            -var "opensuse_microos_arm_expected_sha256=$arm_sha" \
+            hcloud-microos-snapshots.pkr.hcl
+          ;;
+      esac
+    )
+  '';
+
   giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
     cat <<'EOF'
 Gitea runners setup checklist
 
 Tools available in this shell:
-  tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
-  docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
+  tofu, packer, gitea-runners-build-microos-snapshots [x86|arm|both], kubectl, kustomize,
+  kubeconform, sops, age, awscli2, hcloud,
+  tea, docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
 
 Environment expected before real deploy/apply:
   TF_VAR_hcloud_token
@@ -77,6 +266,8 @@ in pkgs.mkShell {
 
   buildInputs = [
     tofu
+    packer
+    buildMicroosSnapshots
     giteaRunnersSetup
     pkgs.nix
     pkgs.kubectl
@@ -95,6 +286,7 @@ in pkgs.mkShell {
     pkgs.curl
     pkgs.git
     pkgs.openssh
+    pkgs.unzip
   ];
 
   shellHook = ''

infra/gitea-runners/k8s/statefulset.yaml

@@ -8,7 +8,7 @@ metadata:
     app.kubernetes.io/part-of: gitea-actions
 spec:
   serviceName: gitea-runner
-  replicas: 5
+  replicas: 1
   podManagementPolicy: Parallel
   selector:
     matchLabels:

infra/gitea-runners/opentofu/README.md

@@ -63,19 +63,25 @@ The default cluster is deliberately fixed-size:
 - cluster name: `gitea-runners`
 - Hetzner location: `fsn1`
 - private network region: `eu-central`
-- control plane: one `cpx21` node in pool `control-plane`
-- workers: three `cpx31` nodes in pool `runner-workers`
+- control plane: one `cpx22` node in pool `control-plane`
+- workers: one `cpx22` node in pool `runner-workers`
 - storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
 - Longhorn: disabled
 - autoscaling/KEDA: not enabled in this stack
 
-The three default workers are sized for the initial five trusted privileged DinD
-jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
-`worker_count` to `5` or increase `worker_server_type`, then run a fresh
-`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
+The default baseline uses one `cpx22` worker to keep the idle bill low while
+still supporting trusted internal jobs. To scale toward higher concurrency
+later, keep autoscaling disabled and either raise `worker_count` or increase
+`worker_server_type`, then run a fresh `tofu plan` and the Task 11 Kubernetes
+pressure checks before applying.
 
 Required inputs must come from environment or secret injection, for example
 `TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
+Set `TF_VAR_firewall_ssh_source` and `TF_VAR_firewall_kube_api_source` explicitly
+to trusted CIDR ranges before planning; the cluster must not expose SSH or port
+6443 to `0.0.0.0/0`.
+If the public key is already registered in Hetzner, set
+`TF_VAR_hcloud_ssh_key_id` to reuse it instead of creating a duplicate key.
 Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
 kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
 enabled; this path is ignored as operational secret material.

infra/gitea-runners/opentofu/backend.tf

@@ -1,10 +1,21 @@
 terraform {
   backend "s3" {
-    bucket       = "gitea-runner-hectic-lab"
-    key          = "gitea-runners/kube-hetzner/terraform.tfstate"
-    region       = "fsn1"
-    encrypt      = true
-    use_lockfile = true
+    bucket = "gitea-runner-hectic-lab"
+    key    = "gitea-runners/kube-hetzner/terraform.tfstate"
+    region = "hel1"
+
+    endpoints = {
+      s3 = "https://hel1.your-objectstorage.com"
+    }
+
+    skip_credentials_validation = true
+    skip_metadata_api_check     = true
+    skip_region_validation      = true
+    skip_requesting_account_id  = true
+    use_path_style              = true
+    encrypt                     = false
+    skip_s3_checksum            = true
+    use_lockfile                = true
   }
 }
 

infra/gitea-runners/opentofu/gitea-runners_kustomization_backup.yaml

@@ -0,0 +1,39 @@
+"apiVersion": "kustomize.config.k8s.io/v1beta1"
+"kind": "Kustomization"
+"patches":
+- "patch": |
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: system-upgrade-controller
+      namespace: system-upgrade
+    spec:
+      template:
+        spec:
+          containers:
+            - name: system-upgrade-controller
+              volumeMounts:
+                - name: ca-certificates
+                  mountPath: /var/lib/ca-certificates
+          volumes:
+            - name: ca-certificates
+              hostPath:
+                path: /var/lib/ca-certificates
+                type: Directory
+  "target":
+    "group": "apps"
+    "kind": "Deployment"
+    "name": "system-upgrade-controller"
+    "namespace": "system-upgrade"
+    "version": "v1"
+- "path": "kured.yaml"
+- "path": "ccm.yaml"
+"resources":
+- "https://github.com/kubereboot/kured/releases/download/1.23.0/kured-1.23.0-combined.yaml"
+- "https://github.com/rancher/system-upgrade-controller/releases/download/v0.18.0/system-upgrade-controller.yaml"
+- "https://github.com/rancher/system-upgrade-controller/releases/download/v0.18.0/crd.yaml"
+- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.35.0/ccm-networks.yaml"
+- "hcloud-csi.yaml"
+- "traefik_ingress.yaml"
+- "flannel-rbac.yaml"
+- "cert_manager.yaml"

infra/gitea-runners/opentofu/main.tf

@@ -47,6 +47,9 @@ module "kube_hetzner" {
   load_balancer_location  = var.hetzner_location
   control_plane_nodepools = local.control_plane_nodepools
   agent_nodepools         = local.agent_nodepools
+  firewall_ssh_source     = var.firewall_ssh_source
+  firewall_kube_api_source = var.firewall_kube_api_source
+  hcloud_ssh_key_id       = var.hcloud_ssh_key_id
 
   # Hetzner CSI is the required StorageClass provider for runner PVCs.
   disable_hetzner_csi = false

infra/gitea-runners/opentofu/variables.tf

@@ -39,21 +39,21 @@ variable "network_region" {
 }
 
 variable "control_plane_server_type" {
-  description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
+  description = "Default control-plane server type. cpx22 is small but leaves headroom for kube-system workloads."
   type        = string
-  default     = "cpx21"
+  default     = "cpx22"
 }
 
 variable "worker_server_type" {
-  description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
+  description = "Default worker server type for the budget trusted DinD runner pool. One cpx22 worker keeps the idle baseline cheap; scale out later if job pressure requires it."
   type        = string
-  default     = "cpx31"
+  default     = "cpx22"
 }
 
 variable "worker_count" {
-  description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
+  description = "Fixed worker count. Default is a single cheap worker for low-cost baseline operation; increase later if concurrency requires it."
   type        = number
-  default     = 3
+  default     = 1
 
   validation {
     condition     = var.worker_count >= 1
@@ -72,3 +72,19 @@ variable "base_domain" {
   type        = string
   default     = ""
 }
+
+variable "firewall_ssh_source" {
+  description = "CIDR ranges allowed to SSH to cluster nodes. Set explicitly; never expose SSH to the public internet."
+  type        = list(string)
+}
+
+variable "firewall_kube_api_source" {
+  description = "CIDR ranges allowed to reach the Kubernetes API. Set explicitly; never expose port 6443 to the public internet."
+  type        = list(string)
+}
+
+variable "hcloud_ssh_key_id" {
+  description = "Existing Hetzner SSH key ID. Set this when the public key is already registered in the project."
+  type        = string
+  default     = null
+}

infra/gitea-runners/runbook.md

@@ -326,7 +326,7 @@ kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
 
 # Run the concurrent smoke workflows now.
 
-kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=1
 kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
 kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
 kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide

nixos/module/hectic/hardware/hetzner-cloud.nix

@@ -132,7 +132,11 @@ in {
           "${cfg.ipv6}::/64"
         ] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
         routes = [
-          { Gateway = "172.31.1.1"; GatewayOnLink = true; }
+          {
+            Gateway = "172.31.1.1";
+            GatewayOnLink = true;
+            PreferredSource = cfg.ipv4;
+          }
           { Gateway = "fe80::1"; }
         ];
       };

sus/gitea-runners.yaml

@@ -2,10 +2,10 @@ gitea:
     hectic-lab:
         org-runner-registration-token: ENC[AES256_GCM,data:Hez1q44P/gxRfpA5Nl1bsf5Be719z/DadWyJMoghaeh+7dX3TqhFmw==,iv:ufme+uMcsI8pw5DcAlxJJKFbcj7XSg2E4MfC12U3sO0=,tag:JLyyBb5PTLY02jOGeB7EEg==,type:str]
 hetzner:
-    hcloud-token: ENC[AES256_GCM,data:uvqCdk+0S3HhSeo/NlGj42ii6Q9aCKq6r0XLuoelBJA=,iv:0sDmmulU4IWIqvhS9n2BTGUutuvZq+8kqW79nBaShBk=,tag:qnQxz5j8sTP9JA5hNFWliw==,type:str]
+    hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str]
 s3:
-    access-key-id: ENC[AES256_GCM,data:fzYW2dI2q43lx7Mb1vMzDAGQeps65ro/0XroLA==,iv:bCmlH4J0Jo0ADSgQ+XxlB4Pjk+1pkmILzYbpfHGyaEs=,tag:m5YtDNOj8u7F1OVtM9NLvQ==,type:str]
-    secret-access-key: ENC[AES256_GCM,data:wxODth7RJNQsmbqr3stAclVAsoeU2BbY94GfkSyORzs=,iv:9QUOyK/2Kc2VuIylsT5d7WOQIiUdwpuzTNvwTW02JBg=,tag:7oY4p1dMzB1A3AeCYYmswA==,type:str]
+    access-key-id: ENC[AES256_GCM,data:o3XiE5HBUfwEMlqX0VdBj5S44w8=,iv:KwPqScuPhTYyr62Ke18fzdMmHlkpaUIL3YoyKUg+F1M=,tag:jz63NLROYIwJ/fJkOIhh+g==,type:str]
+    secret-access-key: ENC[AES256_GCM,data:EhWwJ5jRZ/JURiJwv554g5VE+jHLHK0iXbo3XycLDeeztEHkt9KG9Q==,iv:Y/OSet3iVSaDXRJ+lyl69t64Lm+gVo9DnRFwhYzZ9Cc=,tag:i647yREigMJR+9k4yDPWnA==,type:str]
 sops:
     age:
         - recipient: age1x04u7ftjgx8de2gq596e7frauze764cmn7jjwqnx8szthvfft5qq0tezx6
@@ -53,7 +53,7 @@ sops:
             UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P
             MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-21T23:21:42Z"
-    mac: ENC[AES256_GCM,data:GL21Up+r9f6BcARLFN01BRas1UCVWBGEOVer8wDWVos/UZ8ANUaix1xFqZnwtohF015SRGthfuCLq47B8iPyYeipt2XlRq1VTrCq947Bj+TMTzLqD0nwLsQoFCCGhLPMOF8cKpyZuTbDu3knhBSWUzhDr/1uCf/H0e18shObxTo=,iv:/vqgaN5lL+O6HWBCdoNbk8Bo+T3CaM9DJqMsZrDpl7I=,tag:CmWpAjnBhbh74TtN0+05aQ==,type:str]
+    lastmodified: "2026-08-24T17:35:02Z"
+    mac: ENC[AES256_GCM,data:Nulehy1TdIx4bGCRLWR6JYKqiyadelKG1fuknfPXArjUGzr4q6Pa+SgvBaKL8zcIlu224SxHNUSo3EfPj/O+f5xPH95hhD1vpbZbBBNsls5gMeVMhSELyfhJUn6e7zSwRGzMrY8wZ4VkBRsitHmMTC+Qg33YXsLEm79GKIhzcKI=,iv:zAhpjVhuMsFN/LHmAUGP2/FpsSlBs++bmkktLBxXQnI=,tag:B0wp4ZWfZ6/+ZyBuaKAQHw==,type:str]
     unencrypted_suffix: _unencrypted
-    version: 3.10.2
+    version: 3.12.1