From 1665aaf90ad6b253fb15bdc3002103f2643eede3 Mon Sep 17 00:00:00 2001 From: yukkop Date: Thu, 10 Sep 2026 10:00:28 +0000 Subject: [PATCH] fix: workflow neuro timeouts --- .gitea/workflows/deploy-neuro.yaml | 5 +- docs/attic-cache.md | 33 ++++++-- infra/attic-migration/README.md | 6 +- infra/gitea-runners/runbook.md | 26 +++++++ nixos/system/hectic-lab/attic.nix | 30 +++++++ nixos/system/hectic-lab/hectic-lab.nix | 2 + package/gitea-runner-controller/decide.sh | 2 +- test/package/with-attic-cache/default.nix | 33 +++++++- test/package/with-attic-cache/time-budgets.py | 78 +++++++++++++++++++ 9 files changed, 202 insertions(+), 13 deletions(-) create mode 100644 test/package/with-attic-cache/time-budgets.py diff --git a/.gitea/workflows/deploy-neuro.yaml b/.gitea/workflows/deploy-neuro.yaml index 400b7bd5..d189fffd 100644 --- a/.gitea/workflows/deploy-neuro.yaml +++ b/.gitea/workflows/deploy-neuro.yaml @@ -14,7 +14,7 @@ jobs: name: deploy neuro from performance node if: ${{ gitea.ref == 'refs/heads/master' }} runs-on: gross-nix-x86-perf - timeout-minutes: 60 + timeout-minutes: 435 env: NIX_CONFIG: | fallback = true @@ -39,7 +39,8 @@ jobs: - name: Deploy neuro env: ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} - WITH_ATTIC_BUILD_TIMEOUT: "2700" + WITH_ATTIC_BUILD_TIMEOUT: "21600" + WITH_ATTIC_DRAIN_TIMEOUT: "3600" WITH_ATTIC_UPLOAD_TIMEOUT: "600" WITH_ATTIC_BATCH_SIZE: "8" NEURO_SSH_PRIVATE_KEY: ${{ secrets.NEURO_SSH_PRIVATE_KEY }} diff --git a/docs/attic-cache.md b/docs/attic-cache.md index aed37323..7227d072 100644 --- a/docs/attic-cache.md +++ b/docs/attic-cache.md @@ -196,12 +196,20 @@ minutes for the wrapped command, 10 minutes for the final drain, and three `WITH_ATTIC_UPLOAD_TIMEOUT`, `WITH_ATTIC_UPLOAD_RETRIES`, and `WITH_ATTIC_BATCH_SIZE` (positive integer seconds/counts without leading zeros). -The heavier `deploy-neuro` workflow overrides these defaults: 45 minutes for the -command, batches of at most 8 paths, and 600 seconds per upload attempt. The +The heavier `deploy-neuro` workflow overrides these defaults: 6 hours for the +build/deploy command, batches of at most 8 paths, and 600 seconds per upload attempt. The upload deadline covers the **whole batch**, not each individual path. Its final -drain remains bounded at 10 minutes; the 60-minute job budget leaves 5 minutes -for setup. A prolonged cache outage can still exhaust that drain before every -queued path is uploaded. +drain is bounded at 1 hour; the 435-minute job budget leaves 15 minutes for setup +and cleanup. The `gross-nix-x86-perf` runner limit and Gitea's endless-task +watchdog are 8 hours. The VM lifetime starts at allocation and includes the +controller's additional 10-minute grace. A prolonged cache outage can still +exhaust the drain before every queued path is uploaded. + +The build timeout covers the entire wrapped command, not each derivation. +Completed outputs can be reused from the cache, but an interrupted CUDA/Magma +compilation does not produce a cacheable output or resume on the next ephemeral +runner. Exit code 124 with `interrupted by the user` can therefore mean the +wrapper deadline expired, not that someone manually cancelled the job. The workflow also sets `fallback = true` in `NIX_CONFIG`, inherited by nested Nix commands. If substitution fails, Nix can build the affected derivation from @@ -268,6 +276,21 @@ Usually means: - timeout/reverse proxy issue - bad token permissions +On `hectic-lab`, the upload API has separate nginx locations for +`/_api/v1/upload-path` and `/next/_api/v1/upload-path`. Requests stream to Attic +without whole-body buffering, using HTTP/1.1 upstream and 600-second +`proxy_send_timeout` and `proxy_read_timeout` values. These are inactivity +timeouts, not an upload throughput guarantee. The CI wrapper still enforces its +own whole-batch deadline. The legacy `/previous/` endpoint stays read-only. + +The host's Attic package also restricts its AWS SDK rustls connector to HTTP/1.1 +after observed S3 `REFUSED_STREAM` failures. This is a reproducible, host-scoped +derived Cargo vendor tree; the pinned input tree and Cargo.lock are unchanged. +TLS certificate verification remains enabled, and client-to-nginx HTTP/2 is not +disabled. The pinned crate path makes upstream changes fail visibly during a +future upgrade. This mitigates the observed transport error, not every possible +Hetzner S3 timeout. + ### Cache pulls do not work Check: diff --git a/infra/attic-migration/README.md b/infra/attic-migration/README.md index ebf86252..67233be3 100644 --- a/infra/attic-migration/README.md +++ b/infra/attic-migration/README.md @@ -20,8 +20,10 @@ read availability is fixed. All migration, verification, seeding, and watcher jobs have been stopped for user-controlled load testing. Do not automatically restart bulk verification. -The current generation is -`/nix/store/s7x1n9zprjzagb9pvkl0k4igdgnbbchh-nixos-system-hectic-lab-25.11.20260526.25f5383`. +The transport-fix generation is +`/nix/store/mjx3gzrh03vq22j5nmrzwa56kfhdn702-nixos-system-hectic-lab-25.11.20260526.25f5383` +with the HTTP/1.1 S3 transport and upload-proxy timeout correction. Check +`/run/current-system` to confirm the active deployment. The pinned rollback generation remains at `/var/lib/attic-repack/rollback-system`; backups remain private under `/var/lib/attic-repack/backups` and include the cache signing key. diff --git a/infra/gitea-runners/runbook.md b/infra/gitea-runners/runbook.md index d6d652d1..5280b92e 100644 --- a/infra/gitea-runners/runbook.md +++ b/infra/gitea-runners/runbook.md @@ -557,6 +557,32 @@ Org-level (preferred) or per-repo, on `https://gitea.hectic-lab.com`: - Secret: value of `gitea/hectic-lab/controller/webhook-secret` - Trigger events: `Workflow jobs` only (`workflow_job`) +### Long CUDA/Magma deployment time budgets + +The `deploy-neuro` workflow uses these nested limits: + +| Layer | Limit | +| --- | --- | +| Wrapped build/deploy command | 6 hours (`WITH_ATTIC_BUILD_TIMEOUT=21600`) | +| Final cache drain | 1 hour (`WITH_ATTIC_DRAIN_TIMEOUT=3600`) | +| Workflow job | 435 minutes, including 15 minutes of setup/cleanup margin | +| `gross-nix-x86-perf` runner | 480 minutes | +| Gitea `actions.ENDLESS_TASK_TIMEOUT` | 8 hours | +| VM hard lifetime from allocation | 480 minutes plus 10-minute controller grace | + +Other runner labels keep their existing 180-minute limits. Deploy the controller +and Gitea watchdog settings before dispatching the longer workflow. Already +allocated VMs retain the TTL and runner configuration assigned when they were +created; updating the controller does not extend a running job. + +These are maximum lifetimes: terminal jobs still trigger immediate VM teardown. +The controller's budget reservation uses the full label TTL, so a long-running +label reserves more of the existing monthly budget. Do not raise that budget or +disable timeout safeguards just to bypass a refused allocation. + +After changing any timeout, verify the complete chain rather than only +`timeout-minutes`; a shorter wrapper, runner, server watchdog, or VM TTL wins. + ### Pre-flight verification (before first real job) ```sh diff --git a/nixos/system/hectic-lab/attic.nix b/nixos/system/hectic-lab/attic.nix index 483e16ce..a72abe98 100644 --- a/nixos/system/hectic-lab/attic.nix +++ b/nixos/system/hectic-lab/attic.nix @@ -10,6 +10,14 @@ repackedActive = true; migrationWriteFreeze = false; + uploadProxyConfig = '' + # Stream large NARs and tolerate S3 backpressure while Attic reads them. + proxy_http_version 1.1; + proxy_request_buffering off; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + ''; + repackedSettings = config.services.atticd.settings // { listen = "127.0.0.1:8082"; allowed-hosts = [ "cache.${domain}" ]; @@ -54,6 +62,20 @@ in { # Slow S3 chunk reads can exceed the SDK's default 20-second stall grace. services.atticd.package = pkgs.attic-server.overrideAttrs (old: { + # Restrict the SDK TLS connector to HTTP/1.1 after S3 REFUSED_STREAM errors. + cargoDeps = pkgs.runCommand "attic-cargo-vendor-http1" { } '' + mkdir "$out" + shopt -s dotglob + for entry in ${old.cargoDeps}/*; do + ln -s "$entry" "$out/$(basename "$entry")" + done + crate=aws-smithy-http-client-1.0.6 + rm "$out/$crate" + cp -rL ${old.cargoDeps}/"$crate" "$out/$crate" + chmod -R u+w "$out/$crate" + substituteInPlace "$out/$crate/src/client/tls.rs" \ + --replace-fail '.enable_http2()' "" + ''; postPatch = (old.postPatch or "") + '' substituteInPlace server/src/storage/s3.rs \ --replace-fail 'let mut builder = S3ConfigBuilder::from(&shared_config);' \ @@ -111,6 +133,14 @@ in { proxy_read_timeout 300s; ''; }; + locations."= /_api/v1/upload-path" = lib.mkIf (repackedActive || !migrationWriteFreeze) { + proxyPass = if repackedActive then "http://127.0.0.1:8082" else "http://127.0.0.1:8081"; + extraConfig = uploadProxyConfig; + }; + locations."= /next/_api/v1/upload-path" = { + proxyPass = "http://127.0.0.1:8082/_api/v1/upload-path"; + extraConfig = uploadProxyConfig; + }; locations."/previous/" = { proxyPass = "http://127.0.0.1:8081/"; extraConfig = '' diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index d307d286..453f1bd9 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -355,6 +355,8 @@ in { package = pkgs.hectic.gitea-heatmap; settings.service.DISABLE_REGISTRATION = true; settings.actions.ENABLED = true; + # Long CUDA builds must not hit Gitea's default three-hour task watchdog. + settings.actions.ENDLESS_TASK_TIMEOUT = "8h"; settings.server = { HTTP_PORT = 11011; SSH_PORT = sshPort; diff --git a/package/gitea-runner-controller/decide.sh b/package/gitea-runner-controller/decide.sh index 34c8ab74..f8ae1a69 100644 --- a/package/gitea-runner-controller/decide.sh +++ b/package/gitea-runner-controller/decide.sh @@ -28,7 +28,7 @@ gcr_label_ttl() { gross-mixed-econ) printf '180' ;; gross-nix-x86) printf '180' ;; gross-nix-arm) printf '180' ;; - gross-nix-x86-perf) printf '180' ;; + gross-nix-x86-perf) printf '480' ;; gross-nix-mixed-econ) printf '180' ;; *) return 1 ;; esac diff --git a/test/package/with-attic-cache/default.nix b/test/package/with-attic-cache/default.nix index 9cf5345a..58ff1eaa 100644 --- a/test/package/with-attic-cache/default.nix +++ b/test/package/with-attic-cache/default.nix @@ -45,6 +45,33 @@ if !pkgs.stdenv.hostPlatform.isLinux then {} else let ${builtins.readFile ./launch.sh} mkdir -p "$out" ''; -in lib.mapAttrs' (name: drv: - lib.nameValuePair "with-attic-cache-${name}" (mkTest name drv) -) testDrvs + + timeBudgets = pkgs.runCommand "with-attic-cache-time-budgets" + { + nativeBuildInputs = [ + (pkgs.python3.withPackages (p: [ p.pyyaml ])) + pkgs.dash + ]; + DASH = "${pkgs.dash}/bin/dash"; + WORKFLOW_FILE = ../../../.gitea/workflows/deploy-neuro.yaml; + DECIDE_SH = ../../../package/gitea-runner-controller/decide.sh; + CONTROLLER_SH = ../../../package/gitea-runner-controller/controller.sh; + HCLOUD_SH = ../../../package/gitea-runner-controller/hcloud.sh; + GCR_GITEA_URL = "https://example.invalid"; + GCR_NIX_VERSION = "2.24.0"; + GCR_NIX_TARBALL_SHA256 = "dummy-x86-nix-sha256"; + GCR_ARM_NIX_TARBALL_SHA256 = "dummy-arm-nix-sha256"; + GCR_ACT_RUNNER_VERSION = "0.2.11"; + GCR_ACT_RUNNER_SHA256 = "dummy-runner-sha256"; + GITEA_WATCHDOG = self.nixosConfigurations."hectic-lab|x86_64-linux".config.services.gitea.settings.actions.ENDLESS_TASK_TIMEOUT; + } '' + python ${./time-budgets.py} + mkdir -p "$out" + ''; + + discoveredTests = lib.mapAttrs' (name: drv: + lib.nameValuePair "with-attic-cache-${name}" (mkTest name drv) + ) testDrvs; +in discoveredTests // { + with-attic-cache-time-budgets = timeBudgets; +} diff --git a/test/package/with-attic-cache/time-budgets.py b/test/package/with-attic-cache/time-budgets.py new file mode 100644 index 00000000..165b670b --- /dev/null +++ b/test/package/with-attic-cache/time-budgets.py @@ -0,0 +1,78 @@ +import os +import re +import subprocess + +import yaml + + +def need(condition, message): + if not condition: + raise AssertionError(message) + + +def minutes(value): + text = str(value) + match = re.fullmatch(r"(\d+)([smh])?", text) + if match is None: + raise AssertionError(f"invalid duration: {text}") + amount = int(match.group(1)) + unit = match.group(2) or "m" + return amount // 60 if unit == "s" else amount * 60 if unit == "h" else amount + + +def dash(script): + return subprocess.check_output( + [os.environ["DASH"], "-eu", "-c", script], + text=True, + env=os.environ, + ).strip() + + +def workflow_budget(path): + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + job = data["jobs"]["deploy"] + step = next(s for s in job["steps"] if s.get("name") == "Deploy neuro") + env = step["env"] + return { + "label": job["runs-on"], + "workflow": int(job["timeout-minutes"]), + "build": int(env["WITH_ATTIC_BUILD_TIMEOUT"]) // 60, + "drain": int(env["WITH_ATTIC_DRAIN_TIMEOUT"]) // 60, + "upload": int(env["WITH_ATTIC_UPLOAD_TIMEOUT"]) // 60, + } + + +def shell_budgets(label): + source = f'. "$DECIDE_SH"; . "$CONTROLLER_SH"; . "$HCLOUD_SH"; ' + ttl = int(dash(source + f'gcr_label_ttl "{label}"')) + profile = dash(source + f'gcr_label_profile "{label}"') + grace = int(dash(source + 'gcr_ttl_grace_sec')) // 60 + script = source + f'gcr_bootstrap_script "{label}" token {ttl} runner' + rendered = dash(script) + match = re.search(r"runner:\n(?: .*\n)* timeout: (\d+)m\n", rendered) + if match is None: + raise AssertionError("runner timeout missing in rendered bootstrap") + label_line = f' - "{label}:host"' + need(label_line in rendered, "runner label missing in rendered bootstrap") + labels = "gross-x86 gross-arm gross-x86-perf gross-mixed-econ " \ + "gross-nix-x86 gross-nix-arm gross-nix-mixed-econ" + others = dash(source + f"for l in {labels}; do " + "printf '%s=%s\\n' \"$l\" \"$(gcr_label_ttl \"$l\")\"; done") + return ttl, profile, grace, int(match.group(1)), others.splitlines() + + +budget = workflow_budget(os.environ["WORKFLOW_FILE"]) +need(budget["label"] == "gross-nix-x86-perf", "unexpected deployment runner label") +need(budget["build"] > 0 and budget["drain"] > 0 and budget["upload"] > 0, "non-positive timeout") +need(budget["build"] + budget["drain"] + 15 <= budget["workflow"], "workflow too short for build+drain") + +ttl, profile, grace, runner_timeout, other_ttls = shell_budgets(budget["label"]) +need(ttl == 480, f"expected 480m ttl, got {ttl}") +need(profile.split()[1] == "480", f"profile ttl drifted: {profile}") +need(all(line.endswith("=180") for line in other_ttls), f"default ttl drift: {other_ttls}") +need(budget["workflow"] < runner_timeout, "workflow must be below runner timeout") +need(minutes(os.environ["GITEA_WATCHDOG"]) >= budget["workflow"], "Gitea watchdog too short") +need(ttl + grace >= budget["workflow"] + 15, "VM ttl lacks bootstrap allowance") + +print(f"PASS build={budget['build']}m drain={budget['drain']}m upload={budget['upload']}m workflow={budget['workflow']}m runner={runner_timeout}m ttl={ttl}m grace={grace}m watchdog={os.environ['GITEA_WATCHDOG']}")