diff --git a/.gitea/workflows/deploy-neuro.yaml b/.gitea/workflows/deploy-neuro.yaml index 1a9f321b..400b7bd5 100644 --- a/.gitea/workflows/deploy-neuro.yaml +++ b/.gitea/workflows/deploy-neuro.yaml @@ -17,6 +17,7 @@ jobs: timeout-minutes: 60 env: NIX_CONFIG: | + fallback = true extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA= steps: @@ -31,12 +32,16 @@ jobs: set -eu uname -a nix --version + nix config show fallback nix config show substituters nix config show trusted-public-keys - name: Deploy neuro env: ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + WITH_ATTIC_BUILD_TIMEOUT: "2700" + WITH_ATTIC_UPLOAD_TIMEOUT: "600" + WITH_ATTIC_BATCH_SIZE: "8" NEURO_SSH_PRIVATE_KEY: ${{ secrets.NEURO_SSH_PRIVATE_KEY }} NEURO_SSH_KNOWN_HOSTS: ${{ secrets.NEURO_SSH_KNOWN_HOSTS }} run: | diff --git a/docs/attic-cache.md b/docs/attic-cache.md index d3e1b3be..aed37323 100644 --- a/docs/attic-cache.md +++ b/docs/attic-cache.md @@ -191,11 +191,30 @@ The worker runs during the build and drains after success or failure. Uploads have bounded retries; exhausted uploads fail an otherwise successful command. If the build failed, its original exit status is preserved. Defaults are 30 minutes for the wrapped command, 10 minutes for the final drain, and three -120-second attempts per batch. The workflow allows 60 minutes for setup, the -command, and draining. These limits can be adjusted with +120-second attempts per batch of up to 32 paths. These limits can be adjusted with `WITH_ATTIC_BUILD_TIMEOUT`, `WITH_ATTIC_DRAIN_TIMEOUT`, -`WITH_ATTIC_UPLOAD_TIMEOUT`, and `WITH_ATTIC_UPLOAD_RETRIES` (positive integer -seconds/counts without leading zeros). +`WITH_ATTIC_UPLOAD_TIMEOUT`, `WITH_ATTIC_UPLOAD_RETRIES`, and +`WITH_ATTIC_BATCH_SIZE` (positive integer seconds/counts without leading zeros). + +The heavier `deploy-neuro` workflow overrides these defaults: 45 minutes for the +command, batches of at most 8 paths, and 600 seconds per upload attempt. The +upload deadline covers the **whole batch**, not each individual path. Its final +drain remains bounded at 10 minutes; the 60-minute job budget leaves 5 minutes +for setup. A prolonged cache outage can still exhaust that drain before every +queued path is uploaded. + +The workflow also sets `fallback = true` in `NIX_CONFIG`, inherited by nested +Nix commands. If substitution fails, Nix can build the affected derivation from +source instead of aborting solely because the cache is unavailable. Caches and +signature checks remain enabled. Fallback cannot fix an unavailable upstream +source or a genuine compilation error, and rebuilding can consume more time. + +Uploader logs report each attempt's batch size, whole-batch deadline and exit +status, distinguish deadline expiration from other failures, and list paths in +exhausted batches. The final summary counts queue records: acknowledged, +unconfirmed after exhausted attempts, pending and in flight. These are not +unique artifact counts: an unsuccessful batch may already have uploaded some +paths, and a successful retry can reuse those cached results. This integration targets the root, single-user Nix environment on the ephemeral runner. It refuses to replace an existing post-build hook. SIGINT/SIGTERM stop diff --git a/package/with-attic-cache/with-attic-cache.sh b/package/with-attic-cache/with-attic-cache.sh index 954d0614..1d658d6a 100644 --- a/package/with-attic-cache/with-attic-cache.sh +++ b/package/with-attic-cache/with-attic-cache.sh @@ -30,6 +30,43 @@ queue_empty() { ! ls "$pending_dir"/* >/dev/null 2>&1 && ! ls "$uploading_dir"/* >/dev/null 2>&1 } +count_records() { + dir=$1 + count=0 + for rec in "$dir"/*; do + [ -f "$rec" ] || continue + count=$((count + 1)) + done + printf '%s\n' "$count" +} + +count_failed_records() { + count=0 + for rec in "$failed_dir"/*.tmp; do + [ -f "$rec" ] || continue + count=$((count + 1)) + done + printf '%s\n' "$count" +} + +log_paths_file() { + label=$1 + paths_file=$2 + while IFS= read -r path; do + [ -n "$path" ] || continue + log "$label: $path" + done < "$paths_file" +} + +log_queue_summary() { + acknowledged=$(count_records "$done_dir") + unconfirmed=$(count_failed_records) + pending=$(count_records "$pending_dir") + uploading=$(count_records "$uploading_dir") + log "upload queue summary: acknowledged_records=$acknowledged unconfirmed_records=$unconfirmed pending_records=$pending uploading_records=$uploading" + log "upload queue summary note: acknowledged means attic client accepted a whole batch; unconfirmed failed batches may still contain remote partial successes" +} + write_state() { cat > "$state_file" < "$count_file" [ "$count" -eq 1 ] && exit 9 fi +if [ "${ATTIC_FAIL_MODE:-}" = timeout-once ]; then + count_file="$TEST_ROOT/timeout-count" + count=0 + [ -f "$count_file" ] && count=$(cat "$count_file") + count=$((count + 1)) + printf '%s\n' "$count" > "$count_file" + [ "$count" -eq 1 ] && exit 124 +fi [ "${ATTIC_FAIL_MODE:-}" = permanent ] && exit 10 exit 0 EOS @@ -188,15 +196,32 @@ with-attic-cache -- "$bin/build-command" pass "transient retry" unset ATTIC_FAIL_MODE +make_env +common_env +make_command +export ATTIC_FAIL_MODE=timeout-once +with-attic-cache -- "$bin/build-command" 2> "$root/timeout-once.err" +[ "$(cat "$root/timeout-count")" -eq 2 ] || fail "timeout retry count" +assert_file_contains "timeout attempt exit visible" "$root/timeout-once.err" 'exit=124 class=deadline_timeout' +assert_file_contains "timeout retry logged" "$root/timeout-once.err" 'next_attempt=2/2' +assert_file_contains "timeout retry success logged" "$root/timeout-once.err" 'upload batch accepted: paths=2 attempt=2/2' +assert_file_contains "success final summary" "$root/timeout-once.err" 'acknowledged_records=2 unconfirmed_records=0 pending_records=0 uploading_records=0' +unset ATTIC_FAIL_MODE + make_env common_env make_command export ATTIC_FAIL_MODE=permanent export WITH_ATTIC_DRAIN_TIMEOUT=4 -if with-attic-cache -- "$bin/build-command"; then +if with-attic-cache -- "$bin/build-command" 2> "$root/permanent.err"; then fail "permanent upload failure succeeded" fi [ "$(grep -c '^upload ' "$log")" -le 4 ] || fail "permanent failure retried indefinitely" +assert_file_contains "permanent exhausted path visible" "$root/permanent.err" 'upload exhausted store path: .*/aaa-out' +assert_file_contains "permanent final summary" "$root/permanent.err" 'acknowledged_records=0 unconfirmed_records=2 pending_records=0 uploading_records=0' +if grep -q SECRET "$root/permanent.err"; then + fail "token appeared in permanent stderr" +fi pass "permanent upload failure is nonzero after successful build" unset ATTIC_FAIL_MODE @@ -206,10 +231,12 @@ make_command export ATTIC_FAIL_MODE=permanent export BUILD_EXIT=23 set +e -with-attic-cache -- "$bin/build-command" +with-attic-cache -- "$bin/build-command" 2> "$root/build-fail.err" status=$? set -e [ "$status" -eq 23 ] || fail "build failure status preserved: $status" +assert_file_contains "build failure final summary" "$root/build-fail.err" 'unconfirmed_records=2' +assert_file_contains "build failure exhausted path visible" "$root/build-fail.err" 'upload exhausted store path: .*/bbb-out' pass "build failure status preserved while drain still runs" unset ATTIC_FAIL_MODE BUILD_EXIT @@ -331,9 +358,11 @@ export ATTIC_HANG=1 export WITH_ATTIC_DRAIN_TIMEOUT=1 export WITH_ATTIC_UPLOAD_TIMEOUT=1 export WITH_ATTIC_UPLOAD_RETRIES=1 -if with-attic-cache -- "$bin/build-command"; then +if with-attic-cache -- "$bin/build-command" 2> "$root/drain-timeout.err"; then fail "hung attic returned success" fi +assert_file_contains "drain timeout distinguished" "$root/drain-timeout.err" 'build succeeded but final drain timed out' +assert_file_contains "drain timeout summary" "$root/drain-timeout.err" 'upload queue summary:' if [ -f "$root/attic-grandchild.pid" ]; then child=$(cat "$root/attic-grandchild.pid") i=0