fix: workflow cache

.gitea/workflows/deploy-neuro.yaml

@@ -17,6 +17,7 @@ jobs:
     timeout-minutes: 60
     env:
       NIX_CONFIG: |
+        fallback = true
         extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
         trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
     steps:
@@ -31,12 +32,16 @@ jobs:
           set -eu
           uname -a
           nix --version
+          nix config show fallback
           nix config show substituters
           nix config show trusted-public-keys
 
       - name: Deploy neuro
         env:
           ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+          WITH_ATTIC_BUILD_TIMEOUT: "2700"
+          WITH_ATTIC_UPLOAD_TIMEOUT: "600"
+          WITH_ATTIC_BATCH_SIZE: "8"
           NEURO_SSH_PRIVATE_KEY: ${{ secrets.NEURO_SSH_PRIVATE_KEY }}
           NEURO_SSH_KNOWN_HOSTS: ${{ secrets.NEURO_SSH_KNOWN_HOSTS }}
         run: |

docs/attic-cache.md

@@ -191,11 +191,30 @@ The worker runs during the build and drains after success or failure. Uploads
 have bounded retries; exhausted uploads fail an otherwise successful command.
 If the build failed, its original exit status is preserved. Defaults are 30
 minutes for the wrapped command, 10 minutes for the final drain, and three
-120-second attempts per batch. The workflow allows 60 minutes for setup, the
-command, and draining. These limits can be adjusted with
+120-second attempts per batch of up to 32 paths. These limits can be adjusted with
 `WITH_ATTIC_BUILD_TIMEOUT`, `WITH_ATTIC_DRAIN_TIMEOUT`,
-`WITH_ATTIC_UPLOAD_TIMEOUT`, and `WITH_ATTIC_UPLOAD_RETRIES` (positive integer
-seconds/counts without leading zeros).
+`WITH_ATTIC_UPLOAD_TIMEOUT`, `WITH_ATTIC_UPLOAD_RETRIES`, and
+`WITH_ATTIC_BATCH_SIZE` (positive integer seconds/counts without leading zeros).
+
+The heavier `deploy-neuro` workflow overrides these defaults: 45 minutes for the
+command, batches of at most 8 paths, and 600 seconds per upload attempt. The
+upload deadline covers the **whole batch**, not each individual path. Its final
+drain remains bounded at 10 minutes; the 60-minute job budget leaves 5 minutes
+for setup. A prolonged cache outage can still exhaust that drain before every
+queued path is uploaded.
+
+The workflow also sets `fallback = true` in `NIX_CONFIG`, inherited by nested
+Nix commands. If substitution fails, Nix can build the affected derivation from
+source instead of aborting solely because the cache is unavailable. Caches and
+signature checks remain enabled. Fallback cannot fix an unavailable upstream
+source or a genuine compilation error, and rebuilding can consume more time.
+
+Uploader logs report each attempt's batch size, whole-batch deadline and exit
+status, distinguish deadline expiration from other failures, and list paths in
+exhausted batches. The final summary counts queue records: acknowledged,
+unconfirmed after exhausted attempts, pending and in flight. These are not
+unique artifact counts: an unsuccessful batch may already have uploaded some
+paths, and a successful retry can reuse those cached results.
 
 This integration targets the root, single-user Nix environment on the ephemeral
 runner. It refuses to replace an existing post-build hook. SIGINT/SIGTERM stop

package/with-attic-cache/with-attic-cache.sh

@@ -30,6 +30,43 @@ queue_empty() {
   ! ls "$pending_dir"/* >/dev/null 2>&1 && ! ls "$uploading_dir"/* >/dev/null 2>&1
 }
 
+count_records() {
+  dir=$1
+  count=0
+  for rec in "$dir"/*; do
+    [ -f "$rec" ] || continue
+    count=$((count + 1))
+  done
+  printf '%s\n' "$count"
+}
+
+count_failed_records() {
+  count=0
+  for rec in "$failed_dir"/*.tmp; do
+    [ -f "$rec" ] || continue
+    count=$((count + 1))
+  done
+  printf '%s\n' "$count"
+}
+
+log_paths_file() {
+  label=$1
+  paths_file=$2
+  while IFS= read -r path; do
+    [ -n "$path" ] || continue
+    log "$label: $path"
+  done < "$paths_file"
+}
+
+log_queue_summary() {
+  acknowledged=$(count_records "$done_dir")
+  unconfirmed=$(count_failed_records)
+  pending=$(count_records "$pending_dir")
+  uploading=$(count_records "$uploading_dir")
+  log "upload queue summary: acknowledged_records=$acknowledged unconfirmed_records=$unconfirmed pending_records=$pending uploading_records=$uploading"
+  log "upload queue summary note: acknowledged means attic client accepted a whole batch; unconfirmed failed batches may still contain remote partial successes"
+}
+
 write_state() {
   cat > "$state_file" <<EOF
 attic_bin='$attic_bin'
@@ -143,18 +180,37 @@ fail_claimed() {
 
 upload_once() {
   claim_batch || return 1
+  batch_count=0
+  while IFS= read -r path; do
+    [ -n "$path" ] || continue
+    batch_count=$((batch_count + 1))
+  done < "$batch_file"
   attempt=1
   while [ "$attempt" -le "$upload_retries" ]; do
+    log "upload batch: paths=$batch_count attempt=$attempt/$upload_retries deadline_seconds=$upload_timeout"
     if env -u ATTIC_TOKEN XDG_CONFIG_HOME="$xdg_config_home" \
       "$timeout_bin" --foreground -k 10 "$upload_timeout" \
       "$attic_bin" push --stdin --no-closure --jobs 2 "$attic_cache" \
       < "$batch_file"; then
+      log "upload batch accepted: paths=$batch_count attempt=$attempt/$upload_retries"
       finish_claimed
       return 0
+    else
+      rc=$?
     fi
+    case $rc in
+      124) log "upload batch attempt failed: paths=$batch_count attempt=$attempt/$upload_retries exit=124 class=deadline_timeout" ;;
+      137) log "upload batch attempt failed: paths=$batch_count attempt=$attempt/$upload_retries exit=137 class=forced_kill_or_timeout_kill" ;;
+      *) log "upload batch attempt failed: paths=$batch_count attempt=$attempt/$upload_retries exit=$rc class=attic_exit" ;;
+    esac
     attempt=$((attempt + 1))
-    [ "$attempt" -le "$upload_retries" ] && sleep "$upload_backoff"
+    if [ "$attempt" -le "$upload_retries" ]; then
+      log "upload batch retrying: paths=$batch_count next_attempt=$attempt/$upload_retries backoff_seconds=$upload_backoff"
+      sleep "$upload_backoff"
+    fi
   done
+  log "upload batch exhausted: paths=$batch_count attempts=$upload_retries"
+  log_paths_file "upload exhausted store path" "$batch_file"
   fail_claimed
   return 2
 }
@@ -364,7 +420,12 @@ else
   worker_status=$?
 fi
 [ "$signal_status" -ne 0 ] && build_status=$signal_status
+log_queue_summary
 
+if [ "$build_status" -eq 0 ] && [ "$worker_status" -eq 124 ]; then
+  log "build succeeded but final drain timed out"
+  exit 71
+fi
 if [ "$build_status" -eq 0 ] && [ "$worker_status" -ne 0 ]; then
   log "build succeeded but one or more uploads failed"
   exit 70
@@ -380,4 +441,7 @@ fi
 if [ "$build_status" -ne 0 ] && [ -f "$upload_failed" ]; then
   log "build failed and one or more uploads also failed"
 fi
+if [ "$build_status" -ne 0 ] && [ "$worker_status" -eq 124 ]; then
+  log "build failed and final drain timed out"
+fi
 exit "$build_status"

test/package/with-attic-cache/test/mock-behaviour/run.sh

@@ -74,6 +74,14 @@ if [ "${ATTIC_FAIL_MODE:-}" = transient ]; then
   printf '%s\n' "$count" > "$count_file"
   [ "$count" -eq 1 ] && exit 9
 fi
+if [ "${ATTIC_FAIL_MODE:-}" = timeout-once ]; then
+  count_file="$TEST_ROOT/timeout-count"
+  count=0
+  [ -f "$count_file" ] && count=$(cat "$count_file")
+  count=$((count + 1))
+  printf '%s\n' "$count" > "$count_file"
+  [ "$count" -eq 1 ] && exit 124
+fi
 [ "${ATTIC_FAIL_MODE:-}" = permanent ] && exit 10
 exit 0
 EOS
@@ -188,15 +196,32 @@ with-attic-cache -- "$bin/build-command"
 pass "transient retry"
 unset ATTIC_FAIL_MODE
 
+make_env
+common_env
+make_command
+export ATTIC_FAIL_MODE=timeout-once
+with-attic-cache -- "$bin/build-command" 2> "$root/timeout-once.err"
+[ "$(cat "$root/timeout-count")" -eq 2 ] || fail "timeout retry count"
+assert_file_contains "timeout attempt exit visible" "$root/timeout-once.err" 'exit=124 class=deadline_timeout'
+assert_file_contains "timeout retry logged" "$root/timeout-once.err" 'next_attempt=2/2'
+assert_file_contains "timeout retry success logged" "$root/timeout-once.err" 'upload batch accepted: paths=2 attempt=2/2'
+assert_file_contains "success final summary" "$root/timeout-once.err" 'acknowledged_records=2 unconfirmed_records=0 pending_records=0 uploading_records=0'
+unset ATTIC_FAIL_MODE
+
 make_env
 common_env
 make_command
 export ATTIC_FAIL_MODE=permanent
 export WITH_ATTIC_DRAIN_TIMEOUT=4
-if with-attic-cache -- "$bin/build-command"; then
+if with-attic-cache -- "$bin/build-command" 2> "$root/permanent.err"; then
   fail "permanent upload failure succeeded"
 fi
 [ "$(grep -c '^upload ' "$log")" -le 4 ] || fail "permanent failure retried indefinitely"
+assert_file_contains "permanent exhausted path visible" "$root/permanent.err" 'upload exhausted store path: .*/aaa-out'
+assert_file_contains "permanent final summary" "$root/permanent.err" 'acknowledged_records=0 unconfirmed_records=2 pending_records=0 uploading_records=0'
+if grep -q SECRET "$root/permanent.err"; then
+  fail "token appeared in permanent stderr"
+fi
 pass "permanent upload failure is nonzero after successful build"
 unset ATTIC_FAIL_MODE
 
@@ -206,10 +231,12 @@ make_command
 export ATTIC_FAIL_MODE=permanent
 export BUILD_EXIT=23
 set +e
-with-attic-cache -- "$bin/build-command"
+with-attic-cache -- "$bin/build-command" 2> "$root/build-fail.err"
 status=$?
 set -e
 [ "$status" -eq 23 ] || fail "build failure status preserved: $status"
+assert_file_contains "build failure final summary" "$root/build-fail.err" 'unconfirmed_records=2'
+assert_file_contains "build failure exhausted path visible" "$root/build-fail.err" 'upload exhausted store path: .*/bbb-out'
 pass "build failure status preserved while drain still runs"
 unset ATTIC_FAIL_MODE BUILD_EXIT
 
@@ -331,9 +358,11 @@ export ATTIC_HANG=1
 export WITH_ATTIC_DRAIN_TIMEOUT=1
 export WITH_ATTIC_UPLOAD_TIMEOUT=1
 export WITH_ATTIC_UPLOAD_RETRIES=1
-if with-attic-cache -- "$bin/build-command"; then
+if with-attic-cache -- "$bin/build-command" 2> "$root/drain-timeout.err"; then
   fail "hung attic returned success"
 fi
+assert_file_contains "drain timeout distinguished" "$root/drain-timeout.err" 'build succeeded but final drain timed out'
+assert_file_contains "drain timeout summary" "$root/drain-timeout.err" 'upload queue summary:'
 if [ -f "$root/attic-grandchild.pid" ]; then
   child=$(cat "$root/attic-grandchild.pid")
   i=0