diff --git a/devshell/gitea-runners.nix b/devshell/gitea-runners.nix index a65545bc..b8875563 100644 --- a/devshell/gitea-runners.nix +++ b/devshell/gitea-runners.nix @@ -5,13 +5,202 @@ exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@" ''; + packer = pkgs.stdenvNoCC.mkDerivation { + pname = "packer"; + version = "1.16.0"; + src = pkgs.fetchurl { + url = "https://releases.hashicorp.com/packer/1.16.0/packer_1.16.0_linux_amd64.zip"; + hash = "sha256-XtzRSrWbU1BAxRLb7Nbsnvl2oACwc8Gdk+TEMclIWB4="; + }; + nativeBuildInputs = [ pkgs.unzip ]; + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + installPhase = '' + install -d $out/bin + unzip -p $src packer > $out/bin/packer + chmod 755 $out/bin/packer + ''; + }; + + buildMicroosSnapshots = pkgs.writeShellScriptBin "gitea-runners-build-microos-snapshots" '' + set -eu + + : "''${HCLOUD_TOKEN:?HCLOUD_TOKEN is not set}" + + architecture="''${1:-both}" + x86_location="''${GITEA_RUNNERS_X86_LOCATION:-nbg1}" + x86_server_type="''${GITEA_RUNNERS_X86_SERVER_TYPE:-cx23}" + case "$architecture" in + x86|arm|both) ;; + *) + printf 'usage: gitea-runners-build-microos-snapshots [x86|arm|both]\n' >&2 + exit 2 + ;; + esac + + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT HUP INT TERM + ssh_private_key_file="$workdir/packer-ssh-key" + ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file" + image_public_key_file="''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_ed25519.pub}" + test -r "$image_public_key_file" + packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")" + image_public_key_b64="$(base64 -w0 "$image_public_key_file")" + + curl -fsSL \ + https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh \ + -o "$workdir/create.sh" + chmod +x "$workdir/create.sh" + + ( + cd "$workdir" + folder_name="runner-images" \ + folder_path="$workdir" \ + create_snapshots=none \ + HCLOUD_TOKEN="$HCLOUD_TOKEN" \ + "$workdir/create.sh" + ) + + packer_dir="$workdir/runner-images/packer" + "$packer_dir/scripts/install-verified-packer-plugin-hcloud.sh" + + # Packer may try to remove its remote script after the image-writing reboot, + # while SSH is already unavailable. Keep that cleanup from aborting builds. + sed -i \ + '/inline[[:space:]]*=[[:space:]]*\[local\.write_x86_image\]/a\ skip_clean = true' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + sed -i \ + '/inline[[:space:]]*=[[:space:]]*\[local\.write_arm_image\]/a\ skip_clean = true' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + sed -i \ + '/inline[[:space:]]*=[[:space:]]*\[local\.install_packages\]/a\ start_retry_timeout = "15m"' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + sed -i \ + '/ssh_username[[:space:]]*=[[:space:]]*"root"/a\ temporary_key_pair_type = "ed25519"' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + awk \ + -v ssh_private_key_file="$ssh_private_key_file" \ + '/token[[:space:]]*=[[:space:]]*var[.]hcloud_token/ { + print + print " ssh_private_key_file = \"" ssh_private_key_file "\"" + next + } + { print }' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \ + > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" + mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + + cat > "$workdir/image-key-injection.txt" <<'EOF' + partprobe /dev/sda || true + udevadm settle + root_device="" + for candidate in /dev/sda[0-9]*; do + if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then + root_device="$candidate" + break + fi + done + test -n "$root_device" + mount -o subvol=@ "$root_device" /mnt + install -d -m 0700 /mnt/root/.ssh + printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/authorized_keys + printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/authorized_keys + chmod 0600 /mnt/root/.ssh/authorized_keys + sync + umount /mnt +EOF + sed -i "s|__PACKER_PUBLIC_KEY_B64__|$packer_public_key_b64|" \ + "$workdir/image-key-injection.txt" + sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \ + "$workdir/image-key-injection.txt" + awk -v inject_file="$workdir/image-key-injection.txt" \ + '/done[.] Rebooting/ { + while ((getline line < inject_file) > 0) print line + close(inject_file) + } + { print }' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \ + > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" + mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + + cat > "$workdir/cloud-init-cleanup.txt" <<'EOF' + cloud-init clean --logs --machine-id --seed --configs all || true + rm -rf /run/cloud-init/* /var/lib/cloud/* +EOF + awk -v cleanup_file="$workdir/cloud-init-cleanup.txt" \ + '/# Cleanup some logs/ { + while ((getline line < cleanup_file) > 0) print "- [sh, -c, \"" line "\"]" + close(cleanup_file) + } + { print }' \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \ + > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" + mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \ + "$packer_dir/hcloud-microos-snapshots.pkr.hcl" + + x86_base='https://download.opensuse.org/tumbleweed/appliances' + x86_file='openSUSE-MicroOS.x86_64-ContainerHost-OpenStack-Cloud.qcow2' + x86_url="$x86_base/$x86_file" + + arm_base='https://download.opensuse.org/ports/aarch64/tumbleweed/appliances' + arm_file='openSUSE-MicroOS.aarch64-ContainerHost-OpenStack-Cloud.qcow2' + arm_url="$arm_base/$arm_file" + + x86_sha="$(curl -fsSL "$x86_url.sha256" | awk '{print $1; exit}')" + arm_sha="$(curl -fsSL "$arm_url.sha256" | awk '{print $1; exit}')" + + test -n "$x86_sha" + test -n "$arm_sha" + + printf 'x86 digest: %s\n' "$x86_sha" + printf 'arm digest: %s\n' "$arm_sha" + + ( + cd "$packer_dir" + packer init hcloud-microos-snapshots.pkr.hcl + case "$architecture" in + x86) + packer build \ + -only=hcloud.microos-x86-snapshot \ + -var 'selinux_package_to_install=k3s' \ + -var "x86_location=$x86_location" \ + -var "x86_server_type=$x86_server_type" \ + -var "opensuse_microos_x86_expected_sha256=$x86_sha" \ + -var "opensuse_microos_arm_expected_sha256=$arm_sha" \ + hcloud-microos-snapshots.pkr.hcl + ;; + arm) + packer build \ + -only=hcloud.microos-arm-snapshot \ + -var 'selinux_package_to_install=k3s' \ + -var "opensuse_microos_x86_expected_sha256=$x86_sha" \ + -var "opensuse_microos_arm_expected_sha256=$arm_sha" \ + hcloud-microos-snapshots.pkr.hcl + ;; + both) + packer build \ + -var 'selinux_package_to_install=k3s' \ + -var "x86_location=$x86_location" \ + -var "x86_server_type=$x86_server_type" \ + -var "opensuse_microos_x86_expected_sha256=$x86_sha" \ + -var "opensuse_microos_arm_expected_sha256=$arm_sha" \ + hcloud-microos-snapshots.pkr.hcl + ;; + esac + ) + ''; + giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ '' cat <<'EOF' Gitea runners setup checklist Tools available in this shell: - tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea, - docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix + tofu, packer, gitea-runners-build-microos-snapshots [x86|arm|both], kubectl, kustomize, + kubeconform, sops, age, awscli2, hcloud, + tea, docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix Environment expected before real deploy/apply: TF_VAR_hcloud_token @@ -77,6 +266,8 @@ in pkgs.mkShell { buildInputs = [ tofu + packer + buildMicroosSnapshots giteaRunnersSetup pkgs.nix pkgs.kubectl @@ -95,6 +286,7 @@ in pkgs.mkShell { pkgs.curl pkgs.git pkgs.openssh + pkgs.unzip ]; shellHook = '' diff --git a/infra/gitea-runners/k8s/statefulset.yaml b/infra/gitea-runners/k8s/statefulset.yaml index e9d5bf79..81e330d5 100644 --- a/infra/gitea-runners/k8s/statefulset.yaml +++ b/infra/gitea-runners/k8s/statefulset.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/part-of: gitea-actions spec: serviceName: gitea-runner - replicas: 5 + replicas: 1 podManagementPolicy: Parallel selector: matchLabels: diff --git a/infra/gitea-runners/opentofu/README.md b/infra/gitea-runners/opentofu/README.md index ed56d608..cc6afb24 100644 --- a/infra/gitea-runners/opentofu/README.md +++ b/infra/gitea-runners/opentofu/README.md @@ -63,19 +63,25 @@ The default cluster is deliberately fixed-size: - cluster name: `gitea-runners` - Hetzner location: `fsn1` - private network region: `eu-central` -- control plane: one `cpx21` node in pool `control-plane` -- workers: three `cpx31` nodes in pool `runner-workers` +- control plane: one `cpx22` node in pool `control-plane` +- workers: one `cpx22` node in pool `runner-workers` - storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes` - Longhorn: disabled - autoscaling/KEDA: not enabled in this stack -The three default workers are sized for the initial five trusted privileged DinD -jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise -`worker_count` to `5` or increase `worker_server_type`, then run a fresh -`tofu plan` and the Task 11 Kubernetes pressure checks before applying. +The default baseline uses one `cpx22` worker to keep the idle bill low while +still supporting trusted internal jobs. To scale toward higher concurrency +later, keep autoscaling disabled and either raise `worker_count` or increase +`worker_server_type`, then run a fresh `tofu plan` and the Task 11 Kubernetes +pressure checks before applying. Required inputs must come from environment or secret injection, for example `TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`. +Set `TF_VAR_firewall_ssh_source` and `TF_VAR_firewall_kube_api_source` explicitly +to trusted CIDR ranges before planning; the cluster must not expose SSH or port +6443 to `0.0.0.0/0`. +If the public key is already registered in Hetzner, set +`TF_VAR_hcloud_ssh_key_id` to reuse it instead of creating a duplicate key. Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated kubeconfig to `./_kubeconfig.yaml` when `create_kubeconfig` is enabled; this path is ignored as operational secret material. diff --git a/infra/gitea-runners/opentofu/backend.tf b/infra/gitea-runners/opentofu/backend.tf index 47e95466..83a48a69 100644 --- a/infra/gitea-runners/opentofu/backend.tf +++ b/infra/gitea-runners/opentofu/backend.tf @@ -1,10 +1,21 @@ terraform { backend "s3" { - bucket = "gitea-runner-hectic-lab" - key = "gitea-runners/kube-hetzner/terraform.tfstate" - region = "fsn1" - encrypt = true - use_lockfile = true + bucket = "gitea-runner-hectic-lab" + key = "gitea-runners/kube-hetzner/terraform.tfstate" + region = "hel1" + + endpoints = { + s3 = "https://hel1.your-objectstorage.com" + } + + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_path_style = true + encrypt = false + skip_s3_checksum = true + use_lockfile = true } } diff --git a/infra/gitea-runners/opentofu/gitea-runners_kustomization_backup.yaml b/infra/gitea-runners/opentofu/gitea-runners_kustomization_backup.yaml new file mode 100644 index 00000000..7a06ff0c --- /dev/null +++ b/infra/gitea-runners/opentofu/gitea-runners_kustomization_backup.yaml @@ -0,0 +1,39 @@ +"apiVersion": "kustomize.config.k8s.io/v1beta1" +"kind": "Kustomization" +"patches": +- "patch": | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: system-upgrade-controller + namespace: system-upgrade + spec: + template: + spec: + containers: + - name: system-upgrade-controller + volumeMounts: + - name: ca-certificates + mountPath: /var/lib/ca-certificates + volumes: + - name: ca-certificates + hostPath: + path: /var/lib/ca-certificates + type: Directory + "target": + "group": "apps" + "kind": "Deployment" + "name": "system-upgrade-controller" + "namespace": "system-upgrade" + "version": "v1" +- "path": "kured.yaml" +- "path": "ccm.yaml" +"resources": +- "https://github.com/kubereboot/kured/releases/download/1.23.0/kured-1.23.0-combined.yaml" +- "https://github.com/rancher/system-upgrade-controller/releases/download/v0.18.0/system-upgrade-controller.yaml" +- "https://github.com/rancher/system-upgrade-controller/releases/download/v0.18.0/crd.yaml" +- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.35.0/ccm-networks.yaml" +- "hcloud-csi.yaml" +- "traefik_ingress.yaml" +- "flannel-rbac.yaml" +- "cert_manager.yaml" diff --git a/infra/gitea-runners/opentofu/main.tf b/infra/gitea-runners/opentofu/main.tf index af41f0af..6e0d8ed3 100644 --- a/infra/gitea-runners/opentofu/main.tf +++ b/infra/gitea-runners/opentofu/main.tf @@ -47,6 +47,9 @@ module "kube_hetzner" { load_balancer_location = var.hetzner_location control_plane_nodepools = local.control_plane_nodepools agent_nodepools = local.agent_nodepools + firewall_ssh_source = var.firewall_ssh_source + firewall_kube_api_source = var.firewall_kube_api_source + hcloud_ssh_key_id = var.hcloud_ssh_key_id # Hetzner CSI is the required StorageClass provider for runner PVCs. disable_hetzner_csi = false diff --git a/infra/gitea-runners/opentofu/variables.tf b/infra/gitea-runners/opentofu/variables.tf index 40b5c668..f1f3f318 100644 --- a/infra/gitea-runners/opentofu/variables.tf +++ b/infra/gitea-runners/opentofu/variables.tf @@ -39,21 +39,21 @@ variable "network_region" { } variable "control_plane_server_type" { - description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads." + description = "Default control-plane server type. cpx22 is small but leaves headroom for kube-system workloads." type = string - default = "cpx21" + default = "cpx22" } variable "worker_server_type" { - description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation." + description = "Default worker server type for the budget trusted DinD runner pool. One cpx22 worker keeps the idle baseline cheap; scale out later if job pressure requires it." type = string - default = "cpx31" + default = "cpx22" } variable "worker_count" { - description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack." + description = "Fixed worker count. Default is a single cheap worker for low-cost baseline operation; increase later if concurrency requires it." type = number - default = 3 + default = 1 validation { condition = var.worker_count >= 1 @@ -72,3 +72,19 @@ variable "base_domain" { type = string default = "" } + +variable "firewall_ssh_source" { + description = "CIDR ranges allowed to SSH to cluster nodes. Set explicitly; never expose SSH to the public internet." + type = list(string) +} + +variable "firewall_kube_api_source" { + description = "CIDR ranges allowed to reach the Kubernetes API. Set explicitly; never expose port 6443 to the public internet." + type = list(string) +} + +variable "hcloud_ssh_key_id" { + description = "Existing Hetzner SSH key ID. Set this when the public key is already registered in the project." + type = string + default = null +} diff --git a/infra/gitea-runners/runbook.md b/infra/gitea-runners/runbook.md index 8fdfda22..50249ff5 100644 --- a/infra/gitea-runners/runbook.md +++ b/infra/gitea-runners/runbook.md @@ -326,7 +326,7 @@ kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide # Run the concurrent smoke workflows now. -kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5 +kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=1 kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide diff --git a/nixos/module/hectic/hardware/hetzner-cloud.nix b/nixos/module/hectic/hardware/hetzner-cloud.nix index b2cd2d52..7beedc10 100644 --- a/nixos/module/hectic/hardware/hetzner-cloud.nix +++ b/nixos/module/hectic/hardware/hetzner-cloud.nix @@ -132,7 +132,11 @@ in { "${cfg.ipv6}::/64" ] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32"; routes = [ - { Gateway = "172.31.1.1"; GatewayOnLink = true; } + { + Gateway = "172.31.1.1"; + GatewayOnLink = true; + PreferredSource = cfg.ipv4; + } { Gateway = "fe80::1"; } ]; }; diff --git a/sus/gitea-runners.yaml b/sus/gitea-runners.yaml index 68c6c36e..d7e8b547 100644 --- a/sus/gitea-runners.yaml +++ b/sus/gitea-runners.yaml @@ -2,10 +2,10 @@ gitea: hectic-lab: org-runner-registration-token: ENC[AES256_GCM,data:Hez1q44P/gxRfpA5Nl1bsf5Be719z/DadWyJMoghaeh+7dX3TqhFmw==,iv:ufme+uMcsI8pw5DcAlxJJKFbcj7XSg2E4MfC12U3sO0=,tag:JLyyBb5PTLY02jOGeB7EEg==,type:str] hetzner: - hcloud-token: ENC[AES256_GCM,data:uvqCdk+0S3HhSeo/NlGj42ii6Q9aCKq6r0XLuoelBJA=,iv:0sDmmulU4IWIqvhS9n2BTGUutuvZq+8kqW79nBaShBk=,tag:qnQxz5j8sTP9JA5hNFWliw==,type:str] + hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str] s3: - access-key-id: ENC[AES256_GCM,data:fzYW2dI2q43lx7Mb1vMzDAGQeps65ro/0XroLA==,iv:bCmlH4J0Jo0ADSgQ+XxlB4Pjk+1pkmILzYbpfHGyaEs=,tag:m5YtDNOj8u7F1OVtM9NLvQ==,type:str] - secret-access-key: ENC[AES256_GCM,data:wxODth7RJNQsmbqr3stAclVAsoeU2BbY94GfkSyORzs=,iv:9QUOyK/2Kc2VuIylsT5d7WOQIiUdwpuzTNvwTW02JBg=,tag:7oY4p1dMzB1A3AeCYYmswA==,type:str] + access-key-id: ENC[AES256_GCM,data:o3XiE5HBUfwEMlqX0VdBj5S44w8=,iv:KwPqScuPhTYyr62Ke18fzdMmHlkpaUIL3YoyKUg+F1M=,tag:jz63NLROYIwJ/fJkOIhh+g==,type:str] + secret-access-key: ENC[AES256_GCM,data:EhWwJ5jRZ/JURiJwv554g5VE+jHLHK0iXbo3XycLDeeztEHkt9KG9Q==,iv:Y/OSet3iVSaDXRJ+lyl69t64Lm+gVo9DnRFwhYzZ9Cc=,tag:i647yREigMJR+9k4yDPWnA==,type:str] sops: age: - recipient: age1x04u7ftjgx8de2gq596e7frauze764cmn7jjwqnx8szthvfft5qq0tezx6 @@ -53,7 +53,7 @@ sops: UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-21T23:21:42Z" - mac: ENC[AES256_GCM,data:GL21Up+r9f6BcARLFN01BRas1UCVWBGEOVer8wDWVos/UZ8ANUaix1xFqZnwtohF015SRGthfuCLq47B8iPyYeipt2XlRq1VTrCq947Bj+TMTzLqD0nwLsQoFCCGhLPMOF8cKpyZuTbDu3knhBSWUzhDr/1uCf/H0e18shObxTo=,iv:/vqgaN5lL+O6HWBCdoNbk8Bo+T3CaM9DJqMsZrDpl7I=,tag:CmWpAjnBhbh74TtN0+05aQ==,type:str] + lastmodified: "2026-08-24T17:35:02Z" + mac: ENC[AES256_GCM,data:Nulehy1TdIx4bGCRLWR6JYKqiyadelKG1fuknfPXArjUGzr4q6Pa+SgvBaKL8zcIlu224SxHNUSo3EfPj/O+f5xPH95hhD1vpbZbBBNsls5gMeVMhSELyfhJUn6e7zSwRGzMrY8wZ4VkBRsitHmMTC+Qg33YXsLEm79GKIhzcKI=,iv:zAhpjVhuMsFN/LHmAUGP2/FpsSlBs++bmkktLBxXQnI=,tag:B0wp4ZWfZ6/+ZyBuaKAQHw==,type:str] unencrypted_suffix: _unencrypted - version: 3.10.2 + version: 3.12.1