hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: maintain server until hour
runner ubuntu smoke / ubuntu-latest label smoke (push) Has been cancelled
Details
runner nix smoke / nix label and flake smoke (push) Has been cancelled
Details

Browse Source
This commit is contained in:
yukkop
2026-09-11 02:05:39 +00:00
parent ee8a33c3b0
commit cb62bd3044
13 changed files with 1059 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package/gitea-runner-controller/controller.sh
+261 -66
View File
@@ -3,13 +3,9 @@
# Owns: TTL sweep, orphan-VM sweep, deferred-job retry, stale-runner dereg,
# startup convergence. Runs forever under systemd; webhook service is separate.
gcr_ttl_grace_sec() {
printf '%s' "$((10 * 60))"
}
gcr_record_age_sec() {
created_at="$(gcr_record_field "$1" created_at)"
now="$(date -u '+%s')"
now="$(gcr_now_epoch)"
case "$created_at" in
''|*[!0-9]*) printf '%s' 999999 ;;
*) printf '%s' "$((now - created_at))" ;;
@@ -20,16 +16,63 @@ gcr_sweep_ttl() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
status="$(gcr_record_field "$rec" status)"
[ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue
job_id="$(gcr_record_field "$rec" job_id)"
attempt="$(gcr_record_field "$rec" run_attempt)"
if [ "$status" = "idle_vm" ]; then
gcr_lock_acquire idle-pool || continue
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ -z "$rec" ] || [ "$(gcr_record_field "$rec" status)" != "idle_vm" ]; then
gcr_lock_release idle-pool
continue
fi
if gcr_idle_record_unexpired "$rec"; then
gcr_lock_release idle-pool
continue
fi
vm_id="$(gcr_record_field "$rec" vm_id)"
if gcr_record_vm_owned_elsewhere "$vm_id" "$job_id" "$attempt"; then
gcr_log warn --ns=sweep "removing superseded idle record job=$job_id vm=$vm_id"
gcr_record_del "$job_id" "$attempt"
gcr_lock_release idle-pool
continue
fi
gcr_log info --ns=sweep "idle slot expired job=$job_id vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release idle-pool
continue
fi
[ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue
ttl_min="$(gcr_record_field "$rec" ttl_min)"
case "$ttl_min" in ''|*[!0-9]*) continue ;; esac
max_sec="$((ttl_min * 60 + $(gcr_ttl_grace_sec)))"
max_sec="$((ttl_min * 60))"
age="$(gcr_record_age_sec "$rec")"
if [ "$age" -gt "$max_sec" ]; then
if [ "$age" -ge "$max_sec" ]; then
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
rec="$(gcr_record_get "$job_id" "$attempt")"
case "$(gcr_record_field "$rec" status)" in
pending_vm|vm_active) ;;
*) gcr_lock_release "$key"; continue ;;
esac
ttl_min="$(gcr_record_field "$rec" ttl_min)"
case "$ttl_min" in
''|*[!0-9]*) gcr_lock_release "$key"; continue ;;
esac
max_sec="$((ttl_min * 60))"
age="$(gcr_record_age_sec "$rec")"
if [ "$age" -lt "$max_sec" ]; then
gcr_lock_release "$key"
continue
fi
vm_id="$(gcr_record_field "$rec" vm_id)"
gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
@@ -40,13 +83,21 @@ gcr_sweep_ttl() {
fi
gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}"
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_release "$key"
fi
done
}
gcr_sweep_orphan_vms() {
vms_json="$(gcr_vm_list_managed)" || return 0
gcr_lock_acquire admission || return 0
vms_json="$(gcr_vm_list_managed)" || {
gcr_lock_release admission
return 0
}
if ! gcr_lock_acquire idle-pool; then
gcr_lock_release admission
return 0
fi
count="$(printf '%s' "$vms_json" | jq 'length')"
i=0
while [ "$i" -lt "$count" ]; do
@@ -55,69 +106,125 @@ gcr_sweep_orphan_vms() {
jid="$(printf '%s' "$vm" | jq -r '.labels["gcr.job-id"] // ""')"
att="$(printf '%s' "$vm" | jq -r '.labels["gcr.run-attempt"] // ""')"
known=""
if [ -n "$jid" ] && [ -n "$att" ]; then
rec="$(gcr_record_get "$jid" "$att")"
[ -n "$rec" ] && known=1
fi
if [ -z "$known" ]; then
if ! gcr_record_exists_for_vm_id "$vm_id"; then
gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy"
gcr_vm_destroy "$vm_id" || true
gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}"
fi
i=$((i + 1))
done
gcr_lock_release idle-pool
gcr_lock_release admission
}
gcr_alloc_deferred() {
job_id="$1"; attempt="$2"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || return 0
rec="$(gcr_record_get "$job_id" "$attempt")"
[ -n "$rec" ] || return 0
[ "$(gcr_record_field "$rec" status)" = "deferred" ] || return 0
if [ -z "$rec" ] || [ "$(gcr_record_field "$rec" status)" != "deferred" ]; then
gcr_lock_release "$key"
return 0
fi
repo="$(gcr_record_field "$rec" repo)"
label="$(gcr_record_field "$rec" label)"
state="$(gcr_gitea_job_state "$repo" "$job_id")" || return 0
state="$(gcr_gitea_job_state "$repo" "$job_id")" || {
gcr_lock_release "$key"
return 0
}
case "$state" in
completed:*)
gcr_log info --ns=alloc "deferred job=$job_id already terminal ($state), dropping record"
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
return 0
;;
esac
profile="$(gcr_label_profile "$label")" || return 0
profile="$(gcr_label_profile "$label")" || {
gcr_lock_release "$key"
return 0
}
set -- $profile
server_type="$1"; ttl_min="$2"; rate="$3"
gcr_lock_acquire admission || {
gcr_lock_release "$key"
return 0
}
active="$(gcr_count_active)"
repo_active="$(gcr_count_active_repo "$repo")"
[ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] && return 0
[ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ] && return 0
if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \
|| [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then
gcr_lock_release admission
gcr_lock_release "$key"
return 0
fi
gcr_budget_add "$rate" "$ttl_min" || return 0
reg_token="$(gcr_gitea_registration_token "$repo")" || return 0
claim_status=0
gcr_claim_idle "$job_id" "$attempt" "$repo" "$label" || claim_status="$?"
if [ "$claim_status" -eq 0 ]; then
reused="$(gcr_record_get "$job_id" "$attempt")"
vm_id="$(gcr_record_field "$reused" vm_id)"
if gcr_vm_runner_service "$vm_id" start \
&& gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" false; then
reused="$(gcr_record_get "$job_id" "$attempt")"
reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')"
gcr_record_put "$job_id" "$attempt" "$reused"
else
gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" true || true
gcr_event "vm-reuse-start-failed" "$job_id" "{\"vm_id\":$vm_id,\"via\":\"deferred-retry\"}"
fi
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-reused" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"via\":\"deferred-retry\"}"
gcr_log info --ns=alloc "deferred job=$job_id reused vm=$vm_id"
return 0
fi
if [ "$claim_status" -eq 2 ]; then
gcr_lock_release admission
gcr_lock_release "$key"
return 0
fi
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || return 0
vm_name="gcr-${job_id}-${attempt}"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || {
gcr_budget_can_add "$rate" "$ttl_min" || {
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
vm_name="gcr-${job_id}-${attempt}"
created_at="$(gcr_now_epoch)"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || {
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
gcr_budget_add "$rate" "$ttl_min"
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \
--arg l "$label" --arg t "$created_at" --arg v "$vm_id" \
--arg vn "$vm_name" --arg ttl "$ttl_min" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
gcr_record_put "$job_id" "$attempt" "$rec"
if ! gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_destroy "$vm_id" || true
gcr_lock_release admission
gcr_lock_release "$key"
return 0
fi
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min,\"via\":\"deferred-retry\"}"
gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id"
@@ -134,38 +241,56 @@ gcr_retry_deferred() {
}
gcr_sweep_stale_runners() {
runners="$(gcr_gitea_list_runners)" || return 0
# Here-doc instead of pipe: dash runs pipe tails in a subshell, which
# would strand gcr_event/audit writes from the caller's perspective.
while read -r rid rname; do
[ -n "${rid:-}" ] || continue
case "$rname" in
gcr-*) ;;
gcr_lock_acquire idle-pool || return 0
oldIFS="$IFS"
IFS=,
for allowed_repo in ${GCR_ALLOWED_REPOS:-}; do
IFS="$oldIFS"
case "$allowed_repo" in
*/\*)
owner="${allowed_repo%/*}"
repos="$(gcr_gitea_list_org_repos "$owner")" || {
IFS=,
continue
}
;;
*/*) repos="$allowed_repo" ;;
*) continue ;;
esac
while read -r repo; do
[ -n "${repo:-}" ] || continue
gcr_repo_allowed "$repo" || continue
runners="$(gcr_gitea_list_runners "$repo")" || continue
# Here-doc instead of pipe: dash runs pipe tails in a subshell, which
# would strand gcr_event/audit writes from the caller's perspective.
while read -r rid rname; do
[ -n "${rid:-}" ] || continue
case "$rname" in
gcr-*) ;;
*) continue ;;
esac
# gcr-<job>-<attempt>: alive iff a matching active/pending record exists.
rest="${rname#gcr-}"
jid="${rest%-*}"
att="${rest##*-}"
rec=""
case "$jid" in *[!0-9]*|"") rec="" ;;
*) case "$att" in *[!0-9]*|"") rec="" ;;
*) rec="$(gcr_record_get "$jid" "$att")" ;;
esac ;;
esac
if [ -z "$rec" ]; then
gcr_log warn --ns=sweep "stale runner registration id=$rid name=$rname -> delete"
if gcr_gitea_delete_runner "$rid"; then
gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"runner_id\":$rid,\"name\":\"$rname\"}"
else
gcr_log error --ns=sweep "failed deleting runner id=$rid"
# Runner name stays tied to original VM across later job assignments.
rest="${rname#gcr-}"
jid="${rest%-*}"
if ! gcr_record_exists_for_vm_name "$rname"; then
gcr_log warn --ns=sweep "stale repo=$repo registration id=$rid name=$rname -> delete"
if gcr_gitea_delete_runner "$repo" "$rid"; then
gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"repo\":\"$repo\",\"runner_id\":$rid,\"name\":\"$rname\"}"
else
gcr_log error --ns=sweep "failed deleting repo=$repo runner id=$rid"
fi
fi
fi
done <<EOF
done <<EOF
$runners
EOF
done <<EOF
$repos
EOF
IFS=,
done
IFS="$oldIFS"
gcr_lock_release idle-pool
}
# Runs SSH-push bootstrap for VMs that were created but not yet provisioned.
@@ -178,12 +303,35 @@ gcr_bootstrap_pending() {
job_id="$(gcr_record_field "$rec" job_id)"
attempt="$(gcr_record_field "$rec" run_attempt)"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ "$(gcr_record_field "$rec" status)" != "pending_vm" ] \
|| [ "$(gcr_record_field "$rec" bootstrapped)" = "true" ]; then
gcr_lock_release "$key"
continue
fi
repo="$(gcr_record_field "$rec" repo)"
label="$(gcr_record_field "$rec" label)"
vm_id="$(gcr_record_field "$rec" vm_id)"
runner_name="$(gcr_record_field "$rec" vm_name)"
state="$(gcr_gitea_job_state "$repo" "$job_id")" || continue
if [ "$(gcr_record_field "$rec" reused_vm)" = "true" ]; then
if gcr_vm_runner_service "$vm_id" start \
&& gcr_gitea_runner_disabled "$repo" "$runner_name" false; then
rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | del(.reused_vm)')"
gcr_record_put "$job_id" "$attempt" "$rec"
else
gcr_gitea_runner_disabled "$repo" "$runner_name" true || true
fi
gcr_lock_release "$key"
continue
fi
state="$(gcr_gitea_job_state "$repo" "$job_id")" || {
gcr_lock_release "$key"
continue
}
case "$state" in
completed:*)
gcr_log info --ns=sweep "pending job=$job_id already terminal ($state), destroying vm=$vm_id"
@@ -199,15 +347,24 @@ gcr_bootstrap_pending() {
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_release "$key"
continue
;;
esac
ip="$(gcr_vm_public_ip "$vm_id")"
[ -n "$ip" ] || continue
ip="$(gcr_vm_public_ip "$vm_id")" || {
gcr_lock_release "$key"
continue
}
if [ -z "$ip" ]; then
gcr_lock_release "$key"
continue
fi
reg_token="$(gcr_gitea_registration_token "$repo")" || continue
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_lock_release "$key"
continue
}
ttl_min="$(gcr_record_field "$rec" ttl_min)"
gcr_log info --ns=alloc "bootstrapping vm=$vm_id ip=$ip job=$job_id"
@@ -218,6 +375,7 @@ gcr_bootstrap_pending() {
else
gcr_log warn --ns=alloc "bootstrap failed vm=$vm_id (retry next tick)"
fi
gcr_lock_release "$key"
done
}
@@ -238,6 +396,43 @@ gcr_reap_finished_jobs() {
state="$(gcr_gitea_job_state "$repo" "$job_id")" || continue
case "$state" in
completed:*)
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ -z "$rec" ]; then
gcr_lock_release "$key"
continue
fi
case "$(gcr_record_field "$rec" status)" in
pending_vm|vm_active) ;;
*) gcr_lock_release "$key"; continue ;;
esac
vm_id="$(gcr_record_field "$rec" vm_id)"
if [ "$state" = "completed:success" ] \
&& idle_rec="$(gcr_record_idle_json "$rec")"; then
if ! gcr_lock_acquire idle-pool; then
gcr_lock_release "$key"
continue
fi
runner_name="$(gcr_record_field "$rec" vm_name)"
if ! gcr_gitea_runner_disabled "$repo" "$runner_name" true \
|| ! gcr_vm_runner_service "$vm_id" stop; then
gcr_lock_release idle-pool
gcr_vm_destroy "$vm_id" || true
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
continue
fi
gcr_record_put "$job_id" "$attempt" "$idle_rec"
idle_expires="$(gcr_record_field "$idle_rec" idle_expires_at)"
gcr_lock_release idle-pool
gcr_lock_release "$key"
gcr_event "vm-idle" "$job_id" "{\"vm_id\":$vm_id,\"expires_at\":$idle_expires,\"via\":\"reconcile\"}"
gcr_log info --ns=sweep "job=$job_id succeeded, retaining vm=$vm_id until $idle_expires"
continue
fi
gcr_log info --ns=sweep "job=$job_id terminal ($state), destroying vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
case "$state" in
@@ -251,7 +446,7 @@ gcr_reap_finished_jobs() {
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_release "$key"
;;
esac
done
package/gitea-runner-controller/gitea.sh
+50 -5
View File
@@ -20,12 +20,30 @@ gcr_gitea_admin_token() {
tr -d '\n' < "$GITEA_ADMIN_TOKEN_FILE"
}
# gcr_gitea_list_runners — prints "id name" lines for org hectic-lab.
# gcr_gitea_list_runners REPO — prints "id name" lines for repo runners.
gcr_gitea_list_runners() {
repo="$1"
token="$(gcr_gitea_admin_token)" || return 1
owner="${repo%%/*}"
name="${repo#*/}"
curl -fsS -H "Authorization: token $token" \
"$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners?per_page=50" \
| jq -r '.entries[]? | "\(.id) \(.name)"'
"$GCR_GITEA_URL/api/v1/repos/$owner/$name/actions/runners" \
| jq -r '.runners[]? | "\(.id) \(.name)"'
}
# gcr_gitea_list_org_repos OWNER — prints fully-qualified repository names.
gcr_gitea_list_org_repos() {
owner="$1"
token="$(gcr_gitea_admin_token)" || return 1
page=1
while :; do
repos="$(curl -fsS -H "Authorization: token $token" \
"$GCR_GITEA_URL/api/v1/orgs/$owner/repos?page=$page&limit=50")" || return 1
printf '%s' "$repos" | jq -r '.[]? | .full_name'
count="$(printf '%s' "$repos" | jq 'length')" || return 1
[ "$count" -lt 50 ] && return 0
page=$((page + 1))
done
}
# gcr_gitea_job_state REPO JOB_ID — prints "<status>:<conclusion>".
@@ -40,8 +58,35 @@ gcr_gitea_job_state() {
}
gcr_gitea_delete_runner() {
id="$1"
repo="$1"; id="$2"
token="$(gcr_gitea_admin_token)" || return 1
owner="${repo%%/*}"
name="${repo#*/}"
curl -fsS -X DELETE -H "Authorization: token $token" \
"$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners/$id"
"$GCR_GITEA_URL/api/v1/repos/$owner/$name/actions/runners/$id"
}
gcr_gitea_set_runner_disabled() {
repo="$1"; id="$2"; disabled="$3"
case "$disabled" in true|false) ;; *) return 1 ;; esac
token="$(gcr_gitea_admin_token)" || return 1
owner="${repo%%/*}"
name="${repo#*/}"
curl -fsS -X PATCH -H "Authorization: token $token" \
-H 'Content-Type: application/json' --data "{\"disabled\":$disabled}" \
"$GCR_GITEA_URL/api/v1/repos/$owner/$name/actions/runners/$id" >/dev/null
}
# gcr_gitea_runner_disabled REPO RUNNER_NAME true|false
gcr_gitea_runner_disabled() {
repo="$1"; runner_name="$2"; disabled="$3"
runners="$(gcr_gitea_list_runners "$repo")" || return 1
while read -r id name; do
[ "$name" = "$runner_name" ] || continue
gcr_gitea_set_runner_disabled "$repo" "$id" "$disabled"
return "$?"
done <<EOF
$runners
EOF
return 1
}
package/gitea-runner-controller/hcloud.sh
+26 -5
View File
@@ -92,8 +92,8 @@ labels:
$GCR_DEBUG_SSH_PUBKEY"
fi
# NOTE(yukkop): token reaches only this VM's Hetzner metadata service;
# ephemeral registration makes it useless after the single job exits.
# NOTE(yukkop): token reaches only this VM's Hetzner metadata service and
# is used for initial registration, not for later idle-slot assignments.
printf '%s' "#cloud-config
write_files:
$ssh_key_block
@@ -111,7 +111,7 @@ $(printf '%s\n' "$runner_config" | sed 's/^/ /')
- path: /etc/systemd/system/gitea-runner.service
content: |
[Unit]
Description=Gitea ephemeral Actions runner
Description=Gitea on-demand Actions runner
After=network-online.target gcr-bootstrap.service
Requires=gcr-bootstrap.service
@@ -119,7 +119,7 @@ $(printf '%s\n' "$runner_config" | sed 's/^/ /')
Type=simple
Environment=GITEA_INSTANCE_URL=$GCR_GITEA_URL
Environment=GITEA_RUNNER_REGISTRATION_TOKEN=$reg_token
ExecStart=/usr/local/bin/act_runner daemon --ephemeral --config /etc/gitea-runner/config.yaml
ExecStart=/usr/local/bin/act_runner daemon --config /etc/gitea-runner/config.yaml
Restart=on-failure
RestartSec=5
@@ -232,6 +232,27 @@ gcr_vm_public_ip() {
fi
}
# Stop idle runners so Gitea cannot schedule work before atomic reuse claim.
# The controller starts the service only after the claim record is written.
gcr_vm_runner_service() {
vm_id="$1"; action="$2"
case "$action" in start|stop) ;; *) return 1 ;; esac
ip="$(gcr_vm_public_ip "$vm_id")" || return 1
[ -n "$ip" ] || return 1
test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || return 1
key_tmp="$(mktemp "${TMPDIR:-/tmp}/gcr-runner-sshkey.XXXXXX")"
cat "$GCR_SSH_PRIVKEY_FILE" > "$key_tmp"
printf '\n' >> "$key_tmp"
chmod 0600 "$key_tmp"
ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes"
if timeout 30 ssh $ssh_opts "root@$ip" "systemctl $action gitea-runner.service"; then
rm -f "$key_tmp"
return 0
fi
rm -f "$key_tmp"
return 1
}
gcr_vm_collect_diagnostics() {
vm_id="$1"; ip="$2"; job_id="$3"; reason="$4"
@@ -321,7 +342,7 @@ STARTEOF
chmod 0700 /usr/local/sbin/gcr-runner-start
cat > /etc/systemd/system/gitea-runner.service <<UNITEOF
[Unit]
Description=Gitea ephemeral Actions runner
Description=Gitea on-demand Actions runner
After=network-online.target
[Service]
package/gitea-runner-controller/state.sh
+205 -6
View File
@@ -20,9 +20,54 @@ gcr_record_path() {
printf '%s/jobs/%s.json' "$GCR_STATE_DIR" "$(gcr_alloc_key "$1" "$2")"
}
# mkdir(2) atomicity guard: succeeds exactly once per key until released.
# mkdir(2) atomicity guard. Owner metadata lets a new controller process
# recover locks stranded by a crashed webhook or reconciler process.
gcr_lock_takeover() {
gcr_takeover_dir="$1"
gcr_takeover_old="$gcr_takeover_dir.reclaim.$$"
# Rename is atomic: exactly one reclaimer can move the observed stale
# directory. Never rm -rf the active lock pathname during recovery.
mv "$gcr_takeover_dir" "$gcr_takeover_old" 2>/dev/null || return 1
if mkdir "$gcr_takeover_dir" 2>/dev/null; then
printf '%s %s\n' "$$" "$(gcr_now_epoch)" > "$gcr_takeover_dir/owner"
rm -rf "$gcr_takeover_old"
return 0
fi
rm -rf "$gcr_takeover_old"
return 1
}
gcr_lock_acquire() {
mkdir "$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$1")" 2>/dev/null
gcr_lock_key="$1"
gcr_lock_dir="$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$gcr_lock_key")"
if mkdir "$gcr_lock_dir" 2>/dev/null; then
printf '%s %s\n' "$$" "$(gcr_now_epoch)" > "$gcr_lock_dir/owner"
return 0
fi
gcr_lock_owner="$(cat "$gcr_lock_dir/owner" 2>/dev/null || true)"
if [ -z "$gcr_lock_owner" ]; then
# A crash between mkdir and owner write leaves no PID. Give a live
# creator a short initialization window, then unblock hard TTL work.
gcr_lock_mtime="$(stat -c %Y "$gcr_lock_dir" 2>/dev/null || true)"
gcr_lock_now="$(gcr_now_epoch)"
case "$gcr_lock_mtime:$gcr_lock_now" in
*[!0-9:]*|:*|*::*|*:) return 1 ;;
esac
[ "$((gcr_lock_now - gcr_lock_mtime))" -ge 30 ] || return 1
gcr_lock_takeover "$gcr_lock_dir"
return "$?"
fi
set -- $gcr_lock_owner
gcr_lock_pid="${1:-}"
case "$gcr_lock_pid" in ''|*[!0-9]*) return 1 ;; esac
if kill -0 "$gcr_lock_pid" 2>/dev/null; then
return 1
fi
# Dead PID means a process crash, not live contention. Atomically take
# over its directory; concurrent recovery cannot erase a new lock.
gcr_lock_takeover "$gcr_lock_dir"
}
gcr_lock_release() {
@@ -46,7 +91,151 @@ gcr_record_del() {
}
gcr_record_field() {
printf '%s' "$1" | jq -r --arg f "$2" '.[$f] // ""'
printf '%s' "$1" | jq -r --arg f "$2" 'if has($f) then .[$f] else "" end'
}
gcr_now_epoch() {
date -u '+%s'
}
# Successful VMs remain reusable until next billing-hour boundary, but never
# beyond profile hard TTL. Prints updated idle record when retention is safe.
gcr_record_idle_json() {
gcr_idle_rec="$1"
[ "$(gcr_record_field "$gcr_idle_rec" bootstrapped)" = "true" ] || return 1
gcr_idle_vm_id="$(gcr_record_field "$gcr_idle_rec" vm_id)"
gcr_idle_created="$(gcr_record_field "$gcr_idle_rec" created_at)"
gcr_idle_ttl="$(gcr_record_field "$gcr_idle_rec" ttl_min)"
case "$gcr_idle_vm_id:$gcr_idle_created:$gcr_idle_ttl" in
*[!0-9:]*|0:*|:*|*::*|*:) return 1 ;;
esac
gcr_idle_now="$(gcr_now_epoch)"
gcr_idle_hard_expires="$((gcr_idle_created + gcr_idle_ttl * 60))"
[ "$gcr_idle_now" -lt "$gcr_idle_hard_expires" ] || return 1
gcr_idle_age="$((gcr_idle_now - gcr_idle_created))"
[ "$gcr_idle_age" -ge 0 ] || gcr_idle_age=0
gcr_idle_slots="$((gcr_idle_age / 3600))"
[ "$((gcr_idle_age % 3600))" -eq 0 ] || gcr_idle_slots=$((gcr_idle_slots + 1))
[ "$gcr_idle_slots" -gt 0 ] || gcr_idle_slots=1
gcr_idle_expires="$((gcr_idle_created + gcr_idle_slots * 3600))"
[ "$gcr_idle_expires" -le "$gcr_idle_hard_expires" ] \
|| gcr_idle_expires="$gcr_idle_hard_expires"
printf '%s' "$gcr_idle_rec" | jq -c \
--arg now "$gcr_idle_now" --arg expires "$gcr_idle_expires" \
'.status = "idle_vm"
| .idle_since = ($now | tonumber)
| .idle_expires_at = ($expires | tonumber)'
}
gcr_idle_record_unexpired() {
gcr_idle_rec="$1"
[ "$(gcr_record_field "$gcr_idle_rec" status)" = "idle_vm" ] || return 1
[ "$(gcr_record_field "$gcr_idle_rec" bootstrapped)" = "true" ] || return 1
gcr_idle_now="$(gcr_now_epoch)"
gcr_idle_vm_id="$(gcr_record_field "$gcr_idle_rec" vm_id)"
gcr_idle_vm_name="$(gcr_record_field "$gcr_idle_rec" vm_name)"
gcr_idle_expires="$(gcr_record_field "$gcr_idle_rec" idle_expires_at)"
gcr_idle_created="$(gcr_record_field "$gcr_idle_rec" created_at)"
gcr_idle_ttl="$(gcr_record_field "$gcr_idle_rec" ttl_min)"
case "$gcr_idle_expires:$gcr_idle_created:$gcr_idle_ttl" in
*[!0-9:]*|:*|*::*|*:) return 1 ;;
esac
case "$gcr_idle_vm_id" in ''|0|*[!0-9]*) return 1 ;; esac
[ -n "$gcr_idle_vm_name" ] || return 1
gcr_idle_hard_expires="$((gcr_idle_created + gcr_idle_ttl * 60))"
[ "$gcr_idle_now" -lt "$gcr_idle_expires" ] \
&& [ "$gcr_idle_now" -lt "$gcr_idle_hard_expires" ]
}
gcr_idle_record_usable() {
gcr_idle_rec="$1"
gcr_idle_record_unexpired "$gcr_idle_rec" || return 1
gcr_idle_min_remaining="${GCR_RECONCILE_INTERVAL_SEC:-60}"
case "$gcr_idle_min_remaining" in ''|*[!0-9]*) return 1 ;; esac
[ "$((gcr_idle_expires - gcr_idle_now))" -ge "$gcr_idle_min_remaining" ] \
&& [ "$((gcr_idle_hard_expires - gcr_idle_now))" -ge "$gcr_idle_min_remaining" ]
}
# Caller must hold destination allocation lock. Global pool lock ensures one
# queued job claims an idle VM; destination write precedes source deletion so
# orphan/stale sweeps always see an owner during transfer.
gcr_claim_idle() {
gcr_claim_job="$1"; gcr_claim_attempt="$2"
gcr_claim_repo="$3"; gcr_claim_label="$4"
# Exit 2 means pool is busy; callers must defer instead of charging for a
# new VM without knowing whether matching paid capacity is available.
gcr_lock_acquire idle-pool || return 2
for gcr_claim_file in $(gcr_active_records); do
gcr_claim_rec="$(cat "$gcr_claim_file")"
[ "$(gcr_record_field "$gcr_claim_rec" status)" = "idle_vm" ] || continue
[ "$(gcr_record_field "$gcr_claim_rec" repo)" = "$gcr_claim_repo" ] || continue
[ "$(gcr_record_field "$gcr_claim_rec" label)" = "$gcr_claim_label" ] || continue
gcr_idle_record_usable "$gcr_claim_rec" || continue
gcr_claim_old_job="$(gcr_record_field "$gcr_claim_rec" job_id)"
gcr_claim_old_attempt="$(gcr_record_field "$gcr_claim_rec" run_attempt)"
gcr_claim_vm_id="$(gcr_record_field "$gcr_claim_rec" vm_id)"
gcr_record_vm_owned_elsewhere "$gcr_claim_vm_id" \
"$gcr_claim_old_job" "$gcr_claim_old_attempt" && continue
gcr_claim_now="$(gcr_now_epoch)"
gcr_claim_new="$(printf '%s' "$gcr_claim_rec" | jq -c \
--arg job "$gcr_claim_job" --arg attempt "$gcr_claim_attempt" \
--arg repo "$gcr_claim_repo" --arg label "$gcr_claim_label" \
--arg now "$gcr_claim_now" \
'.job_id = $job | .run_attempt = $attempt
| .repo = $repo | .label = $label | .status = "pending_vm"
| .bootstrapped = false
| .reused_vm = true
| .assigned_at = ($now | tonumber)
| del(.idle_since, .idle_expires_at)')"
if ! gcr_record_put "$gcr_claim_job" "$gcr_claim_attempt" "$gcr_claim_new"; then
gcr_lock_release idle-pool
return 1
fi
gcr_record_del "$gcr_claim_old_job" "$gcr_claim_old_attempt"
gcr_lock_release idle-pool
return 0
done
gcr_lock_release idle-pool
return 1
}
gcr_record_exists_for_vm_id() {
gcr_lookup="$1"
for gcr_lookup_file in $(gcr_active_records); do
[ "$(gcr_record_field "$(cat "$gcr_lookup_file")" vm_id)" = "$gcr_lookup" ] \
&& return 0
done
return 1
}
gcr_record_exists_for_vm_name() {
gcr_lookup="$1"
for gcr_lookup_file in $(gcr_active_records); do
[ "$(gcr_record_field "$(cat "$gcr_lookup_file")" vm_name)" = "$gcr_lookup" ] \
&& return 0
done
return 1
}
gcr_record_vm_owned_elsewhere() {
gcr_lookup_vm="$1"; gcr_lookup_job="$2"; gcr_lookup_attempt="$3"
for gcr_lookup_file in $(gcr_active_records); do
gcr_lookup_rec="$(cat "$gcr_lookup_file")"
[ "$(gcr_record_field "$gcr_lookup_rec" vm_id)" = "$gcr_lookup_vm" ] || continue
if [ "$(gcr_record_field "$gcr_lookup_rec" job_id)" != "$gcr_lookup_job" ] \
|| [ "$(gcr_record_field "$gcr_lookup_rec" run_attempt)" != "$gcr_lookup_attempt" ]; then
return 0
fi
done
return 1
}
gcr_event() {
@@ -55,8 +244,8 @@ gcr_event() {
"$(printf '%s' "$3" | jq -Rs .)" >> "$GCR_STATE_DIR/events.jsonl"
}
# Exit-code contract: 0 = recorded under budget, 1 = would exceed cap.
gcr_budget_add() {
# Exit-code contract: 0 = remains under budget, 1 = would exceed cap.
gcr_budget_can_add() {
rate="$1"; ttl_min="$2"
month="$(date -u '+%Y-%m')"
file="$GCR_STATE_DIR/budget/$month"
@@ -65,10 +254,20 @@ gcr_budget_add() {
if awk -v p="$projected" -v b="${GCR_BUDGET_EUR_MONTHLY:-15}" 'BEGIN {exit !(p > b)}'; then
return 1
fi
return 0
}
# Caller holds admission lock and has already checked gcr_budget_can_add.
gcr_budget_add() {
rate="$1"; ttl_min="$2"
month="$(date -u '+%Y-%m')"
file="$GCR_STATE_DIR/budget/$month"
current="$(cat "$file" 2>/dev/null || echo 0)"
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')"
printf '%s\n' "$projected" > "$file"
return 0
}
gcr_active_records() {
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|deferred)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
}
package/gitea-runner-controller/webhook.sh
+132 -13
View File
@@ -94,6 +94,12 @@ gcr_alloc() {
RESPONSE_CODE=204
return 0
fi
existing="$(gcr_record_get "$job_id" "$attempt")"
if [ -n "$existing" ]; then
gcr_lock_release "$key"
RESPONSE_CODE=204
return 0
fi
if [ "$label_count" -ne 1 ]; then
gcr_lock_release "$key"
@@ -112,24 +118,74 @@ gcr_alloc() {
set -- $profile
server_type="$1"; ttl_min="$2"; rate="$3"
if ! gcr_lock_acquire admission; then
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$(gcr_now_epoch)" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
status:"deferred"}')"
gcr_record_put "$job_id" "$attempt" "$rec"
gcr_lock_release "$key"
gcr_event "deferred" "$job_id" "{\"reason\":\"admission-busy\"}"
RESPONSE_CODE=202; RESPONSE_BODY="deferred: admission busy"
return 0
fi
active="$(gcr_count_active)"
repo_active="$(gcr_count_active_repo "$repo")"
if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \
|| [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$(date -u '+%s')" \
--arg l "$label" --arg t "$(gcr_now_epoch)" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
status:"deferred"}')"
gcr_record_put "$job_id" "$attempt" "$rec"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "deferred" "$job_id" "{\"active\":$active,\"repo_active\":$repo_active}"
RESPONSE_CODE=202; RESPONSE_BODY="deferred: capacity"
return 0
fi
if ! gcr_budget_add "$rate" "$ttl_min"; then
claim_status=0
gcr_claim_idle "$job_id" "$attempt" "$repo" "$label" || claim_status="$?"
if [ "$claim_status" -eq 0 ]; then
reused="$(gcr_record_get "$job_id" "$attempt")"
vm_id="$(gcr_record_field "$reused" vm_id)"
vm_name="$(gcr_record_field "$reused" vm_name)"
if gcr_vm_runner_service "$vm_id" start \
&& gcr_gitea_runner_disabled "$repo" "$vm_name" false; then
reused="$(gcr_record_get "$job_id" "$attempt")"
reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')"
gcr_record_put "$job_id" "$attempt" "$reused"
else
gcr_gitea_runner_disabled "$repo" "$vm_name" true || true
gcr_event "vm-reuse-start-failed" "$job_id" "{\"vm_id\":$vm_id}"
fi
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-reused" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\"}"
RESPONSE_CODE=202; RESPONSE_BODY="reused $vm_name"
return 0
fi
if [ "$claim_status" -eq 2 ]; then
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$(gcr_now_epoch)" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
status:"deferred"}')"
gcr_record_put "$job_id" "$attempt" "$rec"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "deferred" "$job_id" "{\"reason\":\"idle-pool-busy\"}"
RESPONSE_CODE=202; RESPONSE_BODY="deferred: idle pool busy"
return 0
fi
if ! gcr_budget_can_add "$rate" "$ttl_min"; then
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}"
RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted"
@@ -138,6 +194,7 @@ gcr_alloc() {
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "token-error" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="registration token unavailable"
@@ -145,22 +202,33 @@ gcr_alloc() {
}
vm_name="gcr-${job_id}-${attempt}"
created_at="$(gcr_now_epoch)"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || {
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-create-failed" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
return 0
}
gcr_budget_add "$rate" "$ttl_min"
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \
--arg l "$label" --arg t "$created_at" --arg v "$vm_id" \
--arg vn "$vm_name" --arg ttl "$ttl_min" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
gcr_record_put "$job_id" "$attempt" "$rec"
if ! gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_destroy "$vm_id" || true
gcr_lock_release admission
gcr_lock_release "$key"
RESPONSE_CODE=202; RESPONSE_BODY="VM state write failed"
return 0
fi
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min}"
@@ -170,10 +238,48 @@ gcr_alloc() {
gcr_deallocate() {
job_id="$1"; attempt="$2"; new_status="$3"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || return 0
rec="$(gcr_record_get "$job_id" "$attempt")"
[ -n "$rec" ] || return 0
if [ -z "$rec" ]; then
gcr_lock_release "$key"
return 0
fi
case "$(gcr_record_field "$rec" status)" in
pending_vm|vm_active) ;;
*)
gcr_lock_release "$key"
return 0
;;
esac
vm_id="$(gcr_record_field "$rec" vm_id)"
if [ "$new_status" = "completed:success" ] \
&& idle_rec="$(gcr_record_idle_json "$rec")"; then
if ! gcr_lock_acquire idle-pool; then
gcr_lock_release "$key"
return 0
fi
runner_name="$(gcr_record_field "$rec" vm_name)"
if ! gcr_gitea_runner_disabled "$(gcr_record_field "$rec" repo)" "$runner_name" true \
|| ! gcr_vm_runner_service "$vm_id" stop; then
gcr_lock_release idle-pool
gcr_vm_destroy "$vm_id" || true
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
return 0
fi
gcr_record_put "$job_id" "$attempt" "$idle_rec"
idle_expires="$(gcr_record_field "$idle_rec" idle_expires_at)"
gcr_lock_release idle-pool
gcr_lock_release "$key"
gcr_event "vm-idle" "$job_id" "{\"vm_id\":$vm_id,\"expires_at\":$idle_expires}"
return 0
fi
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
case "$new_status" in
completed:success|completed:cancelled|completed:skipped) ;;
@@ -187,7 +293,21 @@ gcr_deallocate() {
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_release "$key"
}
gcr_mark_in_progress() {
job_id="$1"; attempt="$2"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || return 0
rec="$(gcr_record_get "$job_id" "$attempt")"
case "$(gcr_record_field "$rec" status)" in
pending_vm|vm_active)
rec="$(printf '%s' "$rec" | jq -c '.status = "vm_active"')"
gcr_record_put "$job_id" "$attempt" "$rec"
;;
esac
gcr_lock_release "$key"
}
gcr_handle_webhook() {
@@ -210,8 +330,11 @@ gcr_handle_webhook() {
repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')"
labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')"
case "$action:$job_id" in
:*|"queued:"|*":0") gcr_respond 400 "malformed payload"; exit 0 ;;
[ -n "$action" ] || { gcr_respond 400 "malformed payload"; exit 0; }
case "$job_id:$attempt" in
*[!0-9:]*|:*|*::*|*:|0:*|*:0)
gcr_respond 400 "malformed payload"; exit 0
;;
esac
case "$action" in
@@ -220,11 +343,7 @@ gcr_handle_webhook() {
gcr_log info --ns=alloc "queued job=$job_id repo=$repo code=$RESPONSE_CODE $RESPONSE_BODY"
;;
in_progress)
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ -n "$rec" ]; then
rec="$(printf '%s' "$rec" | jq -c '.status = "vm_active"')"
gcr_record_put "$job_id" "$attempt" "$rec"
fi
gcr_mark_in_progress "$job_id" "$attempt"
RESPONSE_CODE=204
;;
completed)