diff --git a/.gitea/workflows/deploy-neuro.yaml b/.gitea/workflows/deploy-neuro.yaml index 188041b8..1a9f321b 100644 --- a/.gitea/workflows/deploy-neuro.yaml +++ b/.gitea/workflows/deploy-neuro.yaml @@ -14,7 +14,7 @@ jobs: name: deploy neuro from performance node if: ${{ gitea.ref == 'refs/heads/master' }} runs-on: gross-nix-x86-perf - timeout-minutes: 30 + timeout-minutes: 60 env: NIX_CONFIG: | extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic @@ -36,6 +36,7 @@ jobs: - name: Deploy neuro env: + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} NEURO_SSH_PRIVATE_KEY: ${{ secrets.NEURO_SSH_PRIVATE_KEY }} NEURO_SSH_KNOWN_HOSTS: ${{ secrets.NEURO_SSH_KNOWN_HOSTS }} run: | @@ -69,5 +70,6 @@ jobs: -i "$HOME/.ssh/id_ed25519" \ neuro true # Run deploy tool from checked-out Gitea revision; repository is not mirrored on GitHub. - nix run --refresh '.#deploy' -- \ - push -- --flake '.#neuro|x86_64-linux' --target-host neuro --use-remote-sudo + nix run --refresh '.#with-attic-cache' -- -- \ + nix run --refresh '.#deploy' -- \ + push -- --flake '.#neuro|x86_64-linux' --target-host neuro --use-remote-sudo diff --git a/docs/attic-cache.md b/docs/attic-cache.md index 61fdb750..d3e1b3be 100644 --- a/docs/attic-cache.md +++ b/docs/attic-cache.md @@ -171,6 +171,56 @@ nix shell nixpkgs#attic-client -c attic cache info local:hectic nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "" ``` +## Automatic uploads from trusted CI + +The `deploy-neuro` workflow uses `with-attic-cache` around its deployment command: + +```sh +# ATTIC_TOKEN must be supplied through a secret, not committed or printed. +nix run '.#with-attic-cache' -- -- nix build '.#my-package' +``` + +The wrapper installs a temporary Nix `post-build-hook`. Each successful local +build queues all output paths, including build-only dependencies and multiple +outputs. A separate worker uploads batches with `attic push --stdin --no-closure` +and two concurrent uploads. Pending outputs have registered garbage-collection +roots until uploaded. Substituted paths and the initial bootstrap of the wrapper +itself are not uploaded; this avoids copying the public NixOS cache into Attic. + +The worker runs during the build and drains after success or failure. Uploads +have bounded retries; exhausted uploads fail an otherwise successful command. +If the build failed, its original exit status is preserved. Defaults are 30 +minutes for the wrapped command, 10 minutes for the final drain, and three +120-second attempts per batch. The workflow allows 60 minutes for setup, the +command, and draining. These limits can be adjusted with +`WITH_ATTIC_BUILD_TIMEOUT`, `WITH_ATTIC_DRAIN_TIMEOUT`, +`WITH_ATTIC_UPLOAD_TIMEOUT`, and `WITH_ATTIC_UPLOAD_RETRIES` (positive integer +seconds/counts without leading zeros). + +This integration targets the root, single-user Nix environment on the ephemeral +runner. It refuses to replace an existing post-build hook. SIGINT/SIGTERM stop +the command and attempt a bounded drain; SIGKILL, VM destruction, or a hard +runner timeout cannot guarantee uploads. A failed upload remains a visible CI +failure, not a claim that the artifact was cached. + +### CI credentials and rollout + +- `ATTIC_TOKEN` is a Gitea repository secret for `hinterland/hearth`, passed only + to the deployment step. The workflow remains manual and restricted to `master`. +- The token grants pull/push only for `hectic`, without deletion or cache + administration. The current token expires **2027-09-09**; rotate it before then. +- The wrapper stores it in a private temporary `0600` file, references that file + from Attic configuration, and removes `ATTIC_TOKEN` from child environments. + Neither the hook nor `NIX_CONFIG` contains the token. Cleanup removes private + files after the worker stops. +- Never expose this credential to untrusted PR workflows or bake it into runner + images. A writer to this cache can publish artifacts trusted by its consumers. +- The `hectic` cache is public for reads. Build outputs must not contain secrets + or content that must remain private; the wrapper uploads every successful local + output, not just the final system. +- Workflow/package changes must be published to `master` before dispatched runs + use them. Creating the secret alone does not enable uploads in an existing run. + ## Common issues ### `flake 'nixpkgs' does not provide attribute 'attic'` diff --git a/nixos/system/hectic-lab/attic.nix b/nixos/system/hectic-lab/attic.nix index 3d5e733d..b6895226 100644 --- a/nixos/system/hectic-lab/attic.nix +++ b/nixos/system/hectic-lab/attic.nix @@ -3,6 +3,7 @@ ... }: { config, + pkgs, ... }: { hectic.services.attic = { @@ -17,6 +18,20 @@ }; }; + # Slow S3 chunk reads can exceed the SDK's default 20-second stall grace. + services.atticd.package = pkgs.attic-server.overrideAttrs (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace server/src/storage/s3.rs \ + --replace-fail 'let mut builder = S3ConfigBuilder::from(&shared_config);' \ + 'let mut builder = S3ConfigBuilder::from(&shared_config) + .stalled_stream_protection( + aws_sdk_s3::config::StalledStreamProtectionConfig::enabled() + .grace_period(Duration::from_secs(120)) + .build(), + );' + ''; + }); + services.nginx.virtualHosts."cache.${domain}" = { enableACME = true; forceSSL = true; @@ -25,6 +40,10 @@ ''; locations."/" = { proxyPass = "http://127.0.0.1:8081"; + extraConfig = '' + # Allow quiet periods while Attic fetches NAR chunks from object storage. + proxy_read_timeout 300s; + ''; }; }; } diff --git a/package/default.nix b/package/default.nix index 9384e916..a300c5b0 100644 --- a/package/default.nix +++ b/package/default.nix @@ -173,4 +173,6 @@ in { pg-15-ext-plsh = buildPlShExt pkgs "15"; stable-video-diffusion-api = pkgs.callPackage ./stable-video-diffusion-api {}; media-browser = pkgs.callPackage ./media-browser {}; +} // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { + with-attic-cache = pkgs.callPackage ./with-attic-cache {}; } diff --git a/package/with-attic-cache/default.nix b/package/with-attic-cache/default.nix new file mode 100644 index 00000000..fb67a1be --- /dev/null +++ b/package/with-attic-cache/default.nix @@ -0,0 +1,38 @@ +{ attic-client +, coreutils +, dash +, gnused +, hectic +, lib +, nix +, util-linux +}: +let + shell = "${dash}/bin/dash"; +in +hectic.writeShellApplication { + inherit shell; + bashOptions = [ + "errexit" + "nounset" + ]; + excludeShellChecks = [ "SC2039" "SC2329" ]; + name = "with-attic-cache"; + runtimeInputs = [ attic-client coreutils gnused nix util-linux ]; + + text = '' + ATTIC_BIN_DEFAULT=${attic-client}/bin/attic + COREUTILS_BIN_DEFAULT=${coreutils}/bin + HOOK_SHELL_DEFAULT=${dash}/bin/dash + NIX_BIN_DEFAULT=${nix}/bin/nix + SETSID_BIN_DEFAULT=${util-linux}/bin/setsid + TIMEOUT_BIN_DEFAULT=${coreutils}/bin/timeout + ${builtins.readFile ./with-attic-cache.sh} + ''; + + meta = { + description = "Run a Nix command while asynchronously uploading new build outputs to Attic"; + mainProgram = "with-attic-cache"; + platforms = lib.platforms.linux; + }; +} diff --git a/package/with-attic-cache/with-attic-cache.sh b/package/with-attic-cache/with-attic-cache.sh new file mode 100644 index 00000000..954d0614 --- /dev/null +++ b/package/with-attic-cache/with-attic-cache.sh @@ -0,0 +1,383 @@ +#!/bin/dash + +die() { + printf '%s\n' "with-attic-cache: $*" >&2 + exit 1 +} + +log() { + printf '%s\n' "with-attic-cache: $*" >&2 +} + +positive_number() { + case ${1:-} in + ''|*[!0-9]*|0*) return 1 ;; + *) return 0 ;; + esac +} + +reject_unsafe_value() { + label=$1 + value=$2 + nl=' +' + case $value in + *"'"*|*"\""*|*"$nl"*) die "$label contains unsupported quote or newline" ;; + esac +} + +queue_empty() { + ! ls "$pending_dir"/* >/dev/null 2>&1 && ! ls "$uploading_dir"/* >/dev/null 2>&1 +} + +write_state() { + cat > "$state_file" < "$hook" <&2; exit 1 ;; + esac + case "\$path" in + *.drv) echo "with-attic-cache hook: refusing drv path: \$path" >&2; exit 1 ;; + esac + [ -e "\$path" ] || { echo "with-attic-cache hook: missing output: \$path" >&2; exit 1; } + base=\$(basename "\$path") + safe=\$(printf '%s' "\$base" | tr -c 'A-Za-z0-9._-' '_') + tmp=\$(mktemp "\$queue_dir/\$safe.XXXXXX.tmp") + id=\$(basename "\$tmp") + rec="\$pending_dir/\$id" + root="\$gcroots_dir/\$id" + "\$nix_bin" build --offline --out-link "\$root" "\$path" >/dev/null + printf '%s\n' "\$path" > "\$tmp" + mv "\$tmp" "\$rec" +done +EOF + chmod 700 "$hook" +} + +write_attic_config() { + mkdir -p "$xdg_config_home/attic" + cat > "$xdg_config_home/attic/config.toml" < "$batch_file" + : > "$claimed_file" + count=0 + for rec in "$pending_dir"/*; do + [ -f "$rec" ] || continue + name=$(basename "$rec") + claimed="$uploading_dir/$name" + if mv "$rec" "$claimed" 2>/dev/null; then + path=$(sed -n '1p' "$claimed") + printf '%s\n' "$path" >> "$batch_file" + printf '%s\n' "$claimed" >> "$claimed_file" + count=$((count + 1)) + [ "$count" -ge "$batch_size" ] && break + fi + done + [ "$count" -gt 0 ] +} + +finish_claimed() { + while IFS= read -r claimed; do + [ -f "$claimed" ] || continue + name=$(basename "$claimed") + mv "$claimed" "$done_dir/$name" 2>/dev/null || rm -f "$claimed" + rm -f "$gcroots_dir/$name" 2>/dev/null || true + done < "$claimed_file" +} + +fail_claimed() { + batch_id=$(date +%s).$$ + cp "$batch_file" "$failed_dir/$batch_id.paths" 2>/dev/null || true + while IFS= read -r claimed; do + [ -f "$claimed" ] || continue + name=$(basename "$claimed") + mv "$claimed" "$failed_dir/$name" 2>/dev/null || true + done < "$claimed_file" + touch "$upload_failed" +} + +upload_once() { + claim_batch || return 1 + attempt=1 + while [ "$attempt" -le "$upload_retries" ]; do + if env -u ATTIC_TOKEN XDG_CONFIG_HOME="$xdg_config_home" \ + "$timeout_bin" --foreground -k 10 "$upload_timeout" \ + "$attic_bin" push --stdin --no-closure --jobs 2 "$attic_cache" \ + < "$batch_file"; then + finish_claimed + return 0 + fi + attempt=$((attempt + 1)) + [ "$attempt" -le "$upload_retries" ] && sleep "$upload_backoff" + done + fail_claimed + return 2 +} + +worker_loop() { + while :; do + upload_once || true + if [ -f "$stop_file" ] && queue_empty; then + break + fi + sleep "$worker_interval" + done + [ ! -f "$upload_failed" ] +} + +# shellcheck disable=SC2317 +stop_build_group() { + [ -n "${build_pid:-}" ] || return 0 + kill -0 "-$build_pid" 2>/dev/null || return 0 + kill -TERM "-$build_pid" 2>/dev/null || kill -TERM "$build_pid" 2>/dev/null || true + i=0 + while kill -0 "-$build_pid" 2>/dev/null && [ "$i" -lt 5 ]; do + sleep 1 + i=$((i + 1)) + done + if kill -0 "-$build_pid" 2>/dev/null; then + kill -KILL "-$build_pid" 2>/dev/null || kill -KILL "$build_pid" 2>/dev/null || true + fi +} + +wait_worker_bounded() { + [ -n "${worker_pid:-}" ] || return 0 + touch "$stop_file" 2>/dev/null || true + i=0 + while [ "$i" -lt "$drain_timeout" ]; do + if ! kill -0 "$worker_pid" 2>/dev/null; then + if wait "$worker_pid" 2>/dev/null; then + kill -KILL "-$worker_pid" 2>/dev/null || true + worker_pid= + return 0 + else + rc=$? + kill -KILL "-$worker_pid" 2>/dev/null || true + worker_pid= + return "$rc" + fi + fi + sleep 1 + i=$((i + 1)) + done + if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then + kill -TERM "-$worker_pid" 2>/dev/null || kill -TERM "$worker_pid" 2>/dev/null || true + sleep 2 + kill -KILL "-$worker_pid" 2>/dev/null || true + wait "$worker_pid" 2>/dev/null || true + worker_pid= + touch "$upload_failed" 2>/dev/null || true + return 124 + fi + return 0 +} + +# shellcheck disable=SC2317 +on_signal() { + signal_status=$1 + stop_build_group + touch "$stop_file" 2>/dev/null || true +} + +# shellcheck disable=SC2317 +cleanup() { + status=$? + trap - EXIT INT TERM + stop_build_group + if [ -n "${worker_pid:-}" ]; then + kill -TERM "-$worker_pid" 2>/dev/null || kill -TERM "$worker_pid" 2>/dev/null || true + sleep 1 + kill -KILL "-$worker_pid" 2>/dev/null || true + wait "$worker_pid" 2>/dev/null || true + fi + rm -rf "$tmp_dir" + exit "$status" +} + +if [ "${1:-}" = "--worker" ]; then + [ "$#" -eq 2 ] || die "usage: with-attic-cache --worker state-file" + # shellcheck disable=SC1090 + . "$2" + trap 'exit 143' TERM + trap 'exit 130' INT + worker_loop + exit $? +fi + +umask 077 +[ "$#" -gt 0 ] || die "usage: with-attic-cache -- command [args...]" +[ "$1" = "--" ] || die "expected -- before command" +shift +[ "$#" -gt 0 ] || die "missing command" +[ -n "${ATTIC_TOKEN:-}" ] || die "ATTIC_TOKEN is required" + +attic_endpoint=${WITH_ATTIC_ENDPOINT:-https://cache.hectic-lab.com} +attic_cache=${WITH_ATTIC_CACHE:-ci:hectic} +build_timeout=${WITH_ATTIC_BUILD_TIMEOUT:-1800} +drain_timeout=${WITH_ATTIC_DRAIN_TIMEOUT:-600} +upload_timeout=${WITH_ATTIC_UPLOAD_TIMEOUT:-120} +upload_retries=${WITH_ATTIC_UPLOAD_RETRIES:-3} +upload_backoff=${WITH_ATTIC_UPLOAD_BACKOFF:-2} +worker_interval=${WITH_ATTIC_WORKER_INTERVAL:-1} +batch_size=${WITH_ATTIC_BATCH_SIZE:-32} + +positive_number "$build_timeout" || die "WITH_ATTIC_BUILD_TIMEOUT must be canonical positive seconds" +positive_number "$drain_timeout" || die "WITH_ATTIC_DRAIN_TIMEOUT must be canonical positive seconds" +positive_number "$upload_timeout" || die "WITH_ATTIC_UPLOAD_TIMEOUT must be canonical positive seconds" +positive_number "$upload_retries" || die "WITH_ATTIC_UPLOAD_RETRIES must be canonical positive" +positive_number "$upload_backoff" || die "WITH_ATTIC_UPLOAD_BACKOFF must be canonical positive seconds" +positive_number "$worker_interval" || die "WITH_ATTIC_WORKER_INTERVAL must be canonical positive seconds" +positive_number "$batch_size" || die "WITH_ATTIC_BATCH_SIZE must be canonical positive" + +reject_unsafe_value WITH_ATTIC_ENDPOINT "$attic_endpoint" +reject_unsafe_value WITH_ATTIC_CACHE "$attic_cache" + +case ${NIX_CONFIG:-} in + *post-build-hook*) die "existing NIX_CONFIG post-build-hook would be replaced; refusing" ;; +esac + +attic_bin=${WITH_ATTIC_ATTIC:-$ATTIC_BIN_DEFAULT} +coreutils_bin=${WITH_ATTIC_COREUTILS_BIN:-$COREUTILS_BIN_DEFAULT} +hook_shell=${WITH_ATTIC_HOOK_SHELL:-$HOOK_SHELL_DEFAULT} +nix_bin=${WITH_ATTIC_NIX:-$NIX_BIN_DEFAULT} +setsid_bin=${WITH_ATTIC_SETSID:-$SETSID_BIN_DEFAULT} +timeout_bin=${WITH_ATTIC_TIMEOUT:-$TIMEOUT_BIN_DEFAULT} +store_dir=${NIX_STORE_DIR:-/nix/store} + +reject_unsafe_value WITH_ATTIC_ATTIC "$attic_bin" +reject_unsafe_value WITH_ATTIC_COREUTILS_BIN "$coreutils_bin" +reject_unsafe_value WITH_ATTIC_HOOK_SHELL "$hook_shell" +reject_unsafe_value WITH_ATTIC_NIX "$nix_bin" +reject_unsafe_value WITH_ATTIC_SETSID "$setsid_bin" +reject_unsafe_value WITH_ATTIC_TIMEOUT "$timeout_bin" +reject_unsafe_value NIX_STORE_DIR "$store_dir" + +configured_hook=$(env -u ATTIC_TOKEN "$nix_bin" config show post-build-hook 2>/dev/null) +[ -z "$configured_hook" ] || die "existing Nix post-build-hook would be replaced; refusing" + +tmp_parent=${TMPDIR:-/tmp} +reject_unsafe_value TMPDIR "$tmp_parent" +tmp_dir=$(mktemp -d "$tmp_parent/with-attic-cache.XXXXXX") +chmod 700 "$tmp_dir" +queue_dir="$tmp_dir/spool" +pending_dir="$queue_dir/pending" +uploading_dir="$queue_dir/uploading" +done_dir="$queue_dir/done" +failed_dir="$queue_dir/failed" +gcroots_dir="$tmp_dir/gcroots" +xdg_config_home="$tmp_dir/xdg" +token_file="$tmp_dir/attic-token" +hook="$tmp_dir/post-build-hook" +batch_file="$tmp_dir/batch.paths" +claimed_file="$tmp_dir/claimed.records" +state_file="$tmp_dir/worker.state" +stop_file="$tmp_dir/stop-worker" +upload_failed="$tmp_dir/upload-failed" +build_pid= +worker_pid= +signal_status=0 +trap cleanup EXIT +trap 'on_signal 130' INT +trap 'on_signal 143' TERM + +mkdir -p "$pending_dir" "$uploading_dir" "$done_dir" "$failed_dir" "$gcroots_dir" "$xdg_config_home" +printf '%s\n' "$ATTIC_TOKEN" > "$token_file" +chmod 600 "$token_file" +unset ATTIC_TOKEN +write_attic_config +make_hook +write_state + +old_nix_config=${NIX_CONFIG:-} +if [ -n "$old_nix_config" ]; then + NIX_CONFIG="$old_nix_config +post-build-hook = $hook" +else + NIX_CONFIG="post-build-hook = $hook" +fi +export NIX_CONFIG + +"$setsid_bin" "$0" --worker "$state_file" & +worker_pid=$! + +"$setsid_bin" "$timeout_bin" -k 15 "$build_timeout" "$@" & +build_pid=$! +if wait "$build_pid"; then + build_status=0 +else + build_status=$? +fi +stop_build_group +build_pid= + +[ "$signal_status" -ne 0 ] && build_status=$signal_status +touch "$stop_file" + +if wait_worker_bounded; then + worker_status=0 +else + worker_status=$? +fi +[ "$signal_status" -ne 0 ] && build_status=$signal_status + +if [ "$build_status" -eq 0 ] && [ "$worker_status" -ne 0 ]; then + log "build succeeded but one or more uploads failed" + exit 70 +fi +if [ "$build_status" -eq 0 ] && [ -f "$upload_failed" ]; then + log "build succeeded but one or more uploads failed" + exit 70 +fi +if [ "$build_status" -eq 0 ] && ! queue_empty; then + log "build succeeded but final drain timed out" + exit 71 +fi +if [ "$build_status" -ne 0 ] && [ -f "$upload_failed" ]; then + log "build failed and one or more uploads also failed" +fi +exit "$build_status" diff --git a/test/package/default.nix b/test/package/default.nix index 56cc45f8..d94b1c93 100644 --- a/test/package/default.nix +++ b/test/package/default.nix @@ -3,6 +3,7 @@ (import ./hemar { inherit system inputs self pkgs; }) // (import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) // (import ./db-tool { inherit system inputs self pkgs; }) // + (import ./with-attic-cache { inherit system inputs self pkgs; }) // (import ./element-web { inherit system inputs self pkgs; }) // (import ./linux-devshell { inherit system inputs self pkgs; }) // (import ./windows-devshell { inherit system inputs self pkgs; }) diff --git a/test/package/with-attic-cache/default.nix b/test/package/with-attic-cache/default.nix new file mode 100644 index 00000000..9cf5345a --- /dev/null +++ b/test/package/with-attic-cache/default.nix @@ -0,0 +1,50 @@ +{ inputs, self, pkgs, system, ... }: +if !pkgs.stdenv.hostPlatform.isLinux then {} else let + lib = inputs.nixpkgs.lib; + + mkTestDrv = name: type: + if type == "directory" then + pkgs.runCommand "test-${name}" {} '' + if ! [ -f ${./test + "/${name}" + /run.sh} ]; then + echo "no run.sh in test/${name}" + exit 1 + fi + mkdir -p "$out" + cp -r ${./test + "/${name}"}/* "$out/" + chmod +x "$out/run.sh" + '' + else if lib.hasSuffix ".sh" name then + pkgs.runCommand "test-${lib.removeSuffix ".sh" name}" {} '' + mkdir -p "$out" + install -Dm755 ${./test + "/${name}"} "$out/run.sh" + '' + else + null; + + testDir = builtins.readDir ./test; + testDrvs = + lib.mapAttrs' (n: v: + lib.nameValuePair (lib.removeSuffix ".sh" n) v + ) (lib.filterAttrs (_: v: v != null) + (lib.mapAttrs (n: t: mkTestDrv n t) testDir)); + + withAtticCache = self.packages.${system}.with-attic-cache; + + mkTest = testName: testDrv: pkgs.runCommand "with-attic-cache-test-${testName}" + { + nativeBuildInputs = [ + pkgs.coreutils + pkgs.dash + pkgs.gnugrep + pkgs.gnused + pkgs.util-linux + withAtticCache + ]; + } '' + test=${testDrv} + ${builtins.readFile ./launch.sh} + mkdir -p "$out" + ''; +in lib.mapAttrs' (name: drv: + lib.nameValuePair "with-attic-cache-${name}" (mkTest name drv) +) testDrvs diff --git a/test/package/with-attic-cache/launch.sh b/test/package/with-attic-cache/launch.sh new file mode 100644 index 00000000..95ca021c --- /dev/null +++ b/test/package/with-attic-cache/launch.sh @@ -0,0 +1,22 @@ +#!/bin/dash +set -eu + +fail() { + printf '%s\n' "FAIL: $*" >&2 + exit 1 +} + +pass() { + printf '%s\n' "PASS: $*" >&2 +} + +assert_file_contains() { + label=$1 + file=$2 + pattern=$3 + grep -q "$pattern" "$file" || fail "$label: missing $pattern in $file" + pass "$label" +} + +# shellcheck disable=SC1091,SC2154 +. "$test/run.sh" diff --git a/test/package/with-attic-cache/test/mock-behaviour/run.sh b/test/package/with-attic-cache/test/mock-behaviour/run.sh new file mode 100644 index 00000000..23dc32d3 --- /dev/null +++ b/test/package/with-attic-cache/test/mock-behaviour/run.sh @@ -0,0 +1,369 @@ +#!/bin/dash +set -eu + +make_env() { + root=$(mktemp -d) + store="$root/store" + bin="$root/bin" + log="$root/log" + test_shell=$(command -v dash) + mkdir -p "$store" "$bin" + : > "$log" + + printf '#!%s\n' "$test_shell" > "$bin/nix" + cat >> "$bin/nix" <<'EOS' +set -eu +if [ "$1" = config ] && [ "$2" = show ] && [ "$3" = post-build-hook ]; then + [ "${NIX_CONFIG_FAIL:-}" = 1 ] && exit 12 + [ "${NIX_CONFIGURED_HOOK:-}" = 1 ] && printf '/configured/hook\n' + exit 0 +fi +if [ "$1" = build ] && [ "$2" = --offline ] && [ "$3" = --out-link ]; then + root=$4 + path=$5 + mkdir -p "$(dirname "$root")" + ln -s "$path" "$root" + printf 'gcroot %s -> %s\n' "$root" "$path" >> "$TEST_LOG" + exit 0 +fi +exit 2 +EOS + chmod +x "$bin/nix" + + printf '#!%s\n' "$test_shell" > "$bin/attic" + cat >> "$bin/attic" <<'EOS' +set -eu +[ "${ATTIC_TOKEN+x}" ] && { echo token leaked to attic env >&2; exit 41; } +case "$*" in *SECRET*) echo token leaked to argv >&2; exit 42 ;; esac +config="$XDG_CONFIG_HOME/attic/config.toml" +grep -q 'token-file = ' "$config" || exit 43 +! grep -q SECRET "$config" || exit 44 +token_file=$(sed -n 's/token-file = "\(.*\)"/\1/p' "$config") +mode=$(stat -c '%a' "$token_file") +[ "$mode" = 600 ] || { echo "bad token mode $mode" >&2; exit 45; } +[ "$(cat "$token_file")" = SECRET ] || exit 46 +tmp_root=$(dirname "$XDG_CONFIG_HOME") +while IFS= read -r path; do + [ -n "$path" ] || continue + case "$path" in *.drv) echo drv queued >&2; exit 47 ;; esac + found=0 + tries=0 + while [ "$tries" -lt 20 ]; do + for root in "$tmp_root/gcroots"/*; do + [ -L "$root" ] || continue + [ "$(readlink "$root")" = "$path" ] && found=1 + done + [ "$found" -eq 1 ] && break + tries=$((tries + 1)) + sleep 0.1 + done + [ "$found" -eq 1 ] || { echo "missing gcroot for $path" >&2; exit 48; } + printf 'upload %s\n' "$path" >> "$TEST_LOG" +done +if [ "${ATTIC_HANG:-}" = 1 ]; then + trap '' TERM + sleep 60 & + printf '%s\n' "$!" > "$TEST_ROOT/attic-grandchild.pid" + wait +fi +if [ "${ATTIC_FAIL_MODE:-}" = transient ]; then + count_file="$TEST_ROOT/transient-count" + count=0 + [ -f "$count_file" ] && count=$(cat "$count_file") + count=$((count + 1)) + printf '%s\n' "$count" > "$count_file" + [ "$count" -eq 1 ] && exit 9 +fi +[ "${ATTIC_FAIL_MODE:-}" = permanent ] && exit 10 +exit 0 +EOS + chmod +x "$bin/attic" +} + +common_env() { + export ATTIC_TOKEN=SECRET + export NIX_STORE_DIR="$store" + export TEST_LOG="$log" + export TEST_ROOT="$root" + export WITH_ATTIC_ATTIC="$bin/attic" + export WITH_ATTIC_NIX="$bin/nix" + export WITH_ATTIC_BUILD_TIMEOUT=5 + export WITH_ATTIC_DRAIN_TIMEOUT=3 + export WITH_ATTIC_UPLOAD_TIMEOUT=2 + export WITH_ATTIC_UPLOAD_RETRIES=2 + export WITH_ATTIC_UPLOAD_BACKOFF=1 + export WITH_ATTIC_WORKER_INTERVAL=1 + export WITH_ATTIC_BATCH_SIZE=64 +} + +make_command() { + printf '#!%s\n' "$test_shell" > "$bin/build-command" + cat >> "$bin/build-command" <<'EOS' +set -eu +[ "${ATTIC_TOKEN+x}" ] && { echo token leaked to build env >&2; exit 31; } +hook=$(printf '%s\n' "$NIX_CONFIG" | sed -n 's/^post-build-hook = //p') +[ -x "$hook" ] || exit 32 +mkdir -p "$NIX_STORE_DIR/aaa-out" "$NIX_STORE_DIR/bbb-out" "$NIX_STORE_DIR/ccc-out.drv" +OUT_PATHS="$NIX_STORE_DIR/aaa-out $NIX_STORE_DIR/bbb-out" "$hook" +if OUT_PATHS="$NIX_STORE_DIR/ccc-out.drv" "$hook" 2>/dev/null; then + echo drv accepted >&2 + exit 33 +fi +printf 'build ok\n' >> "$TEST_LOG" +exit "${BUILD_EXIT:-0}" +EOS + chmod +x "$bin/build-command" +} + +make_concurrent_command() { + printf '#!%s\n' "$test_shell" > "$bin/build-command" + cat >> "$bin/build-command" <<'EOS' +set -eu +hook=$(printf '%s\n' "$NIX_CONFIG" | sed -n 's/^post-build-hook = //p') +for n in 1 2 3 4 5; do + mkdir -p "$NIX_STORE_DIR/out-$n" + OUT_PATHS="$NIX_STORE_DIR/out-$n" "$hook" & +done +wait +EOS + chmod +x "$bin/build-command" +} + +make_slow_command() { + printf '#!%s\n' "$test_shell" > "$bin/build-command" + cat >> "$bin/build-command" <<'EOS' +trap 'printf terminated >> "$TEST_LOG"; exit 99' TERM +sleep 10 +EOS + chmod +x "$bin/build-command" +} + +make_signal_command() { + printf '#!%s\n' "$test_shell" > "$bin/build-command" + cat >> "$bin/build-command" <<'EOS' +set -eu +hook=$(printf '%s\n' "$NIX_CONFIG" | sed -n 's/^post-build-hook = //p') +mkdir -p "$NIX_STORE_DIR/signal-out" +OUT_PATHS="$NIX_STORE_DIR/signal-out" "$hook" +trap '' TERM +sleep 60 & +printf '%s\n' "$!" > "$TEST_ROOT/build-grandchild.pid" +wait +EOS + chmod +x "$bin/build-command" +} + +make_env +common_env +make_command +with-attic-cache -- "$bin/build-command" +assert_file_contains "multiple outputs uploaded" "$log" 'upload .*/aaa-out' +assert_file_contains "space separated outputs uploaded" "$log" 'upload .*/bbb-out' +assert_file_contains "hook excluded drv path" "$log" 'build ok' +assert_file_contains "gcroot registered" "$log" 'gcroot .*aaa-out' + +make_env +common_env +make_command +with-attic-cache -- "$bin/build-command" +if grep -q SECRET "$log"; then + fail "token appeared in test log" +fi +pass "token absent from command logs" + +make_env +common_env +make_concurrent_command +with-attic-cache -- "$bin/build-command" +uploads=$(grep -c '^upload ' "$log") +[ "$uploads" -eq 5 ] || fail "concurrent producers uploaded $uploads paths, expected 5" +pass "concurrent producers" + +make_env +common_env +make_command +export ATTIC_FAIL_MODE=transient +with-attic-cache -- "$bin/build-command" +[ "$(cat "$root/transient-count")" -eq 2 ] || fail "transient retry count" +pass "transient retry" +unset ATTIC_FAIL_MODE + +make_env +common_env +make_command +export ATTIC_FAIL_MODE=permanent +export WITH_ATTIC_DRAIN_TIMEOUT=4 +if with-attic-cache -- "$bin/build-command"; then + fail "permanent upload failure succeeded" +fi +[ "$(grep -c '^upload ' "$log")" -le 4 ] || fail "permanent failure retried indefinitely" +pass "permanent upload failure is nonzero after successful build" +unset ATTIC_FAIL_MODE + +make_env +common_env +make_command +export ATTIC_FAIL_MODE=permanent +export BUILD_EXIT=23 +set +e +with-attic-cache -- "$bin/build-command" +status=$? +set -e +[ "$status" -eq 23 ] || fail "build failure status preserved: $status" +pass "build failure status preserved while drain still runs" +unset ATTIC_FAIL_MODE BUILD_EXIT + +make_env +common_env +make_command +unset ATTIC_TOKEN +if with-attic-cache -- "$bin/build-command" 2> "$root/missing.err"; then + fail "missing token accepted" +fi +assert_file_contains "missing token rejected" "$root/missing.err" 'ATTIC_TOKEN is required' + +make_env +common_env +make_slow_command +export WITH_ATTIC_BUILD_TIMEOUT=1 +set +e +with-attic-cache -- "$bin/build-command" +status=$? +set -e +[ "$status" -ne 0 ] || fail "timeout command succeeded" +pass "bounded timeout returns nonzero" + +make_env +common_env +make_command +export WITH_ATTIC_BUILD_TIMEOUT=0 +if with-attic-cache -- "$bin/build-command" 2> "$root/knob.err"; then + fail "invalid timeout accepted" +fi +assert_file_contains "positive number validation" "$root/knob.err" 'WITH_ATTIC_BUILD_TIMEOUT must be canonical positive' + +make_env +common_env +make_command +export WITH_ATTIC_BUILD_TIMEOUT=00 +if with-attic-cache -- "$bin/build-command" 2> "$root/zero.err"; then + fail "all-zero timeout accepted" +fi +assert_file_contains "all-zero rejected" "$root/zero.err" 'canonical positive' + +make_env +common_env +make_command +export WITH_ATTIC_BUILD_TIMEOUT=08 +if with-attic-cache -- "$bin/build-command" 2> "$root/octal.err"; then + fail "leading-zero timeout accepted" +fi +assert_file_contains "leading zero rejected" "$root/octal.err" 'canonical positive' + +make_env +common_env +make_command +export NIX_CONFIG='post-build-hook = /already/configured' +if with-attic-cache -- "$bin/build-command" 2> "$root/hook.err"; then + fail "existing hook accepted" +fi +assert_file_contains "existing hook refused" "$root/hook.err" 'existing NIX_CONFIG post-build-hook' + +make_env +common_env +make_command +unset NIX_CONFIG +export NIX_CONFIGURED_HOOK=1 +if with-attic-cache -- "$bin/build-command" 2> "$root/config-hook.err"; then + fail "configured hook accepted" +fi +assert_file_contains "configured hook refused" "$root/config-hook.err" 'existing Nix post-build-hook' +unset NIX_CONFIGURED_HOOK + +make_env +common_env +make_command +export NIX_CONFIG_FAIL=1 +if with-attic-cache -- "$bin/build-command" 2> "$root/config-fail.err"; then + fail "nix config failure ignored" +fi +unset NIX_CONFIG_FAIL +pass "nix config failure is fatal" + +make_env +common_env +make_command +export WITH_ATTIC_ENDPOINT='https://cache.example/"bad"' +if with-attic-cache -- "$bin/build-command" 2> "$root/quote.err"; then + fail "unsafe endpoint accepted" +fi +assert_file_contains "unsafe endpoint rejected" "$root/quote.err" 'unsupported quote or newline' +unset WITH_ATTIC_ENDPOINT + +make_env +common_env +make_signal_command +export WITH_ATTIC_BUILD_TIMEOUT=30 +export WITH_ATTIC_DRAIN_TIMEOUT=5 +with-attic-cache -- "$bin/build-command" & +wrapper=$! +i=0 +while ! grep -q 'upload .*/signal-out' "$log" && [ "$i" -lt 20 ]; do + sleep 0.2 + i=$((i + 1)) +done +kill -TERM "$wrapper" +set +e +wait "$wrapper" +status=$? +set -e +[ "$status" -eq 143 ] || fail "TERM status $status, expected 143" +assert_file_contains "TERM drains queued output" "$log" 'upload .*/signal-out' +if [ -f "$root/build-grandchild.pid" ] && kill -0 "$(cat "$root/build-grandchild.pid")" 2>/dev/null; then + fail "build grandchild survived TERM cleanup" +fi +pass "TERM cleanup kills build descendants" + +make_env +common_env +make_command +export ATTIC_HANG=1 +export WITH_ATTIC_DRAIN_TIMEOUT=1 +export WITH_ATTIC_UPLOAD_TIMEOUT=1 +export WITH_ATTIC_UPLOAD_RETRIES=1 +if with-attic-cache -- "$bin/build-command"; then + fail "hung attic returned success" +fi +if [ -f "$root/attic-grandchild.pid" ]; then + child=$(cat "$root/attic-grandchild.pid") + i=0 + while kill -0 "$child" 2>/dev/null && [ "$i" -lt 20 ]; do + sleep 0.1 + i=$((i + 1)) + done + if kill -0 "$child" 2>/dev/null; then + fail "attic grandchild survived worker cleanup" + fi +fi +pass "hung attic descendants cleaned" +unset ATTIC_HANG + +make_env +common_env +real_setsid=$(command -v setsid) +printf '#!%s\n' "$test_shell" > "$bin/setsid" +cat >> "$bin/setsid" <