549 lines
21 KiB
Bash
549 lines
21 KiB
Bash
#!/bin/dash
|
|
# Reconcile loop for gitea-runner-controller.
|
|
# Owns: TTL sweep, orphan-VM sweep, deferred-job retry, stale-runner dereg,
|
|
# startup convergence. Runs forever under systemd; webhook service is separate.
|
|
|
|
gcr_record_age_sec() {
|
|
created_at="$(gcr_record_field "$1" created_at)"
|
|
now="$(gcr_now_epoch)"
|
|
case "$created_at" in
|
|
''|*[!0-9]*) printf '%s' 999999 ;;
|
|
*) printf '%s' "$((now - created_at))" ;;
|
|
esac
|
|
}
|
|
|
|
gcr_sweep_ttl() {
|
|
for f in $(gcr_active_records); do
|
|
rec="$(cat "$f")"
|
|
status="$(gcr_record_field "$rec" status)"
|
|
job_id="$(gcr_record_field "$rec" job_id)"
|
|
attempt="$(gcr_record_field "$rec" run_attempt)"
|
|
|
|
if [ "$status" = "idle_vm" ]; then
|
|
gcr_lock_acquire idle-pool || continue
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
if [ -z "$rec" ] || [ "$(gcr_record_field "$rec" status)" != "idle_vm" ]; then
|
|
gcr_lock_release idle-pool
|
|
continue
|
|
fi
|
|
if gcr_idle_record_unexpired "$rec"; then
|
|
gcr_lock_release idle-pool
|
|
continue
|
|
fi
|
|
|
|
vm_id="$(gcr_record_field "$rec" vm_id)"
|
|
if gcr_record_vm_owned_elsewhere "$vm_id" "$job_id" "$attempt"; then
|
|
gcr_log warn --ns=sweep "removing superseded idle record job=$job_id vm=$vm_id"
|
|
gcr_record_del "$job_id" "$attempt"
|
|
gcr_lock_release idle-pool
|
|
continue
|
|
fi
|
|
gcr_log info --ns=sweep "idle slot expired job=$job_id vm=$vm_id"
|
|
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
|
|
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-expired false; then
|
|
gcr_event "vm-destroyed" "$job_id" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
|
|
else
|
|
gcr_event "vm-cleanup-pending" "$job_id" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
|
|
fi
|
|
else
|
|
gcr_record_del "$job_id" "$attempt"
|
|
fi
|
|
gcr_lock_release idle-pool
|
|
continue
|
|
fi
|
|
|
|
[ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue
|
|
ttl_min="$(gcr_record_field "$rec" ttl_min)"
|
|
case "$ttl_min" in ''|*[!0-9]*) continue ;; esac
|
|
|
|
max_sec="$((ttl_min * 60))"
|
|
age="$(gcr_record_age_sec "$rec")"
|
|
if [ "$age" -ge "$max_sec" ]; then
|
|
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
|
gcr_lock_acquire "$key" || continue
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
case "$(gcr_record_field "$rec" status)" in
|
|
pending_vm|vm_active) ;;
|
|
*) gcr_lock_release "$key"; continue ;;
|
|
esac
|
|
ttl_min="$(gcr_record_field "$rec" ttl_min)"
|
|
case "$ttl_min" in
|
|
''|*[!0-9]*) gcr_lock_release "$key"; continue ;;
|
|
esac
|
|
max_sec="$((ttl_min * 60))"
|
|
age="$(gcr_record_age_sec "$rec")"
|
|
if [ "$age" -lt "$max_sec" ]; then
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
|
|
vm_id="$(gcr_record_field "$rec" vm_id)"
|
|
gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s"
|
|
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
|
|
ip="$(gcr_vm_public_ip "$vm_id" || true)"
|
|
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true
|
|
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" ttl false; then
|
|
gcr_event "vm-destroyed" "$job_id" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
|
|
else
|
|
gcr_event "vm-cleanup-pending" "$job_id" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
|
|
fi
|
|
else
|
|
gcr_record_del "$job_id" "$attempt"
|
|
fi
|
|
gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}"
|
|
gcr_lock_release "$key"
|
|
fi
|
|
done
|
|
}
|
|
|
|
gcr_sweep_orphan_vms() {
|
|
gcr_lock_acquire admission || return 0
|
|
vms_json="$(gcr_vm_list_managed)" || {
|
|
gcr_lock_release admission
|
|
return 0
|
|
}
|
|
if ! gcr_lock_acquire idle-pool; then
|
|
gcr_lock_release admission
|
|
return 0
|
|
fi
|
|
count="$(printf '%s' "$vms_json" | jq 'length')"
|
|
i=0
|
|
while [ "$i" -lt "$count" ]; do
|
|
vm="$(printf '%s' "$vms_json" | jq -c ".[$i]")"
|
|
vm_id="$(printf '%s' "$vm" | jq -r '.id')"
|
|
jid="$(printf '%s' "$vm" | jq -r '.labels["gcr.job-id"] // ""')"
|
|
att="$(printf '%s' "$vm" | jq -r '.labels["gcr.run-attempt"] // ""')"
|
|
|
|
if ! gcr_record_exists_for_vm_id "$vm_id"; then
|
|
gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy"
|
|
cleanup_job="${jid:-orphan-$vm_id}"
|
|
cleanup_attempt="${att:-0}"
|
|
cleanup_rec="$(jq -n --arg j "$cleanup_job" --arg a "$cleanup_attempt" \
|
|
--arg v "$vm_id" --arg vn "$(printf '%s' "$vm" | jq -r '.name // ""')" \
|
|
'{job_id:$j, run_attempt:$a, repo:"", label:"", created_at:"0",
|
|
ttl_min:0, vm_id:($v|tonumber), vm_name:$vn,
|
|
bootstrapped:false, status:"cleanup_pending"}')"
|
|
if gcr_vm_cleanup_start "$cleanup_job" "$cleanup_attempt" \
|
|
"$cleanup_rec" orphan false; then
|
|
gcr_event "orphan-vm-destroyed" "$cleanup_job" "{\"vm_id\":$vm_id}"
|
|
else
|
|
gcr_event "vm-cleanup-pending" "$cleanup_job" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"orphan\"}"
|
|
fi
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
gcr_lock_release idle-pool
|
|
gcr_lock_release admission
|
|
}
|
|
|
|
gcr_alloc_deferred() {
|
|
job_id="$1"; attempt="$2"
|
|
|
|
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
|
gcr_lock_acquire "$key" || return 0
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
if [ -z "$rec" ] || [ "$(gcr_record_field "$rec" status)" != "deferred" ]; then
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
fi
|
|
|
|
repo="$(gcr_record_field "$rec" repo)"
|
|
label="$(gcr_record_field "$rec" label)"
|
|
|
|
state="$(gcr_gitea_job_state "$repo" "$job_id")" || {
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
}
|
|
case "$state" in
|
|
completed:*)
|
|
gcr_log info --ns=alloc "deferred job=$job_id already terminal ($state), dropping record"
|
|
gcr_record_del "$job_id" "$attempt"
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
;;
|
|
esac
|
|
|
|
profile="$(gcr_label_profile "$label")" || {
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
}
|
|
set -- $profile
|
|
server_type="$1"; ttl_min="$2"
|
|
|
|
gcr_lock_acquire admission || {
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
}
|
|
active="$(gcr_count_active)"
|
|
repo_active="$(gcr_count_active_repo "$repo")"
|
|
if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \
|
|
|| [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
fi
|
|
|
|
claim_status=0
|
|
gcr_claim_idle "$job_id" "$attempt" "$repo" "$label" || claim_status="$?"
|
|
if [ "$claim_status" -eq 0 ]; then
|
|
reused="$(gcr_record_get "$job_id" "$attempt")"
|
|
vm_id="$(gcr_record_field "$reused" vm_id)"
|
|
if gcr_vm_runner_service "$vm_id" start \
|
|
&& gcr_vm_runner_service "$vm_id" health \
|
|
&& gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" false; then
|
|
reused="$(gcr_record_get "$job_id" "$attempt")"
|
|
reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')"
|
|
gcr_record_put "$job_id" "$attempt" "$reused"
|
|
else
|
|
gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" true || true
|
|
gcr_event "vm-reuse-start-failed" "$job_id" "{\"vm_id\":$vm_id,\"via\":\"deferred-retry\"}"
|
|
fi
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
gcr_event "vm-reused" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"via\":\"deferred-retry\"}"
|
|
gcr_log info --ns=alloc "deferred job=$job_id reused vm=$vm_id"
|
|
return 0
|
|
fi
|
|
if [ "$claim_status" -eq 2 ]; then
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
fi
|
|
|
|
reg_token="$(gcr_gitea_registration_token "$repo")" || {
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
}
|
|
|
|
vm_name="gcr-${job_id}-${attempt}"
|
|
created_at="$(gcr_now_epoch)"
|
|
create_status=0
|
|
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
|
|
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
|
|
if [ "$create_status" -ne 0 ] || [ -z "$created" ]; then
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
fi
|
|
set -- $created
|
|
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
|
|
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
|
|
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
|
|
"$actual_server_type" "$actual_rate"; then
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
return 0
|
|
fi
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min,\"via\":\"deferred-retry\"}"
|
|
gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id"
|
|
}
|
|
|
|
gcr_sweep_cleanup_pending() {
|
|
for f in $(gcr_active_records); do
|
|
rec="$(cat "$f")"
|
|
[ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ] || continue
|
|
job_id="$(gcr_record_field "$rec" job_id)"
|
|
attempt="$(gcr_record_field "$rec" run_attempt)"
|
|
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
|
gcr_lock_acquire "$key" || continue
|
|
if ! gcr_lock_acquire admission; then
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
if [ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ]; then
|
|
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
|
|
fi
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
done
|
|
}
|
|
|
|
gcr_sweep_create_ambiguous() {
|
|
for f in $(gcr_active_records); do
|
|
rec="$(cat "$f")"
|
|
[ "$(gcr_record_field "$rec" status)" = "create_ambiguous" ] || continue
|
|
job_id="$(gcr_record_field "$rec" job_id)"
|
|
attempt="$(gcr_record_field "$rec" run_attempt)"
|
|
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
|
gcr_lock_acquire "$key" || continue
|
|
if ! gcr_lock_acquire admission; then
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
if [ "$(gcr_record_field "$rec" status)" != "create_ambiguous" ]; then
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
find_status=0
|
|
found_vm_id="$(gcr_vm_find_created \
|
|
"$(gcr_record_field "$rec" vm_name)" "$job_id" "$attempt" \
|
|
"$(gcr_record_field "$rec" label)" \
|
|
"$(gcr_record_field "$rec" server_type)" \
|
|
"$(gcr_record_field "$rec" candidate_location)" \
|
|
"$(gcr_record_field "$rec" candidate_arch)")" || find_status="$?"
|
|
case "$find_status" in
|
|
0)
|
|
rec="$(printf '%s' "$rec" | jq -c --arg vm "$found_vm_id" \
|
|
'.vm_id = ($vm | tonumber)
|
|
| .status = "pending_vm"
|
|
| .bootstrapped = false
|
|
| del(.create_http, .create_curl_status,
|
|
.candidate_location, .candidate_arch)')"
|
|
if gcr_record_put "$job_id" "$attempt" "$rec"; then
|
|
gcr_event "vm-create-recovered" "$job_id" \
|
|
"{\"vm_id\":$found_vm_id,\"label\":\"$(gcr_record_field "$rec" label)\"}"
|
|
fi
|
|
;;
|
|
1)
|
|
rec="$(printf '%s' "$rec" | jq -c \
|
|
'.status = "cleanup_pending"
|
|
| .cleanup_reason = "ambiguous-create-absent"
|
|
| .cleanup_refund_budget = true
|
|
| .cleanup_vm_destroyed = true
|
|
| .cleanup_budget_released = false')"
|
|
if gcr_record_put "$job_id" "$attempt" "$rec"; then
|
|
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
|
|
fi
|
|
;;
|
|
2) ;;
|
|
esac
|
|
gcr_lock_release admission
|
|
gcr_lock_release "$key"
|
|
done
|
|
}
|
|
|
|
gcr_retry_deferred() {
|
|
for f in $(gcr_active_records); do
|
|
rec="$(cat "$f")"
|
|
[ "$(gcr_record_field "$rec" status)" = "deferred" ] || continue
|
|
gcr_alloc_deferred \
|
|
"$(gcr_record_field "$rec" job_id)" \
|
|
"$(gcr_record_field "$rec" run_attempt)"
|
|
done
|
|
}
|
|
|
|
gcr_sweep_stale_runners() {
|
|
gcr_lock_acquire idle-pool || return 0
|
|
oldIFS="$IFS"
|
|
IFS=,
|
|
for allowed_repo in ${GCR_ALLOWED_REPOS:-}; do
|
|
IFS="$oldIFS"
|
|
case "$allowed_repo" in
|
|
*/\*)
|
|
owner="${allowed_repo%/*}"
|
|
repos="$(gcr_gitea_list_org_repos "$owner")" || {
|
|
IFS=,
|
|
continue
|
|
}
|
|
;;
|
|
*/*) repos="$allowed_repo" ;;
|
|
*) continue ;;
|
|
esac
|
|
while read -r repo; do
|
|
[ -n "${repo:-}" ] || continue
|
|
gcr_repo_allowed "$repo" || continue
|
|
runners="$(gcr_gitea_list_runners "$repo")" || continue
|
|
# Here-doc instead of pipe: dash runs pipe tails in a subshell, which
|
|
# would strand gcr_event/audit writes from the caller's perspective.
|
|
while read -r rid rname; do
|
|
[ -n "${rid:-}" ] || continue
|
|
case "$rname" in
|
|
gcr-*) ;;
|
|
*) continue ;;
|
|
esac
|
|
|
|
# Runner name stays tied to original VM across later job assignments.
|
|
rest="${rname#gcr-}"
|
|
jid="${rest%-*}"
|
|
if ! gcr_record_exists_for_vm_name "$rname"; then
|
|
gcr_log warn --ns=sweep "stale repo=$repo registration id=$rid name=$rname -> delete"
|
|
if gcr_gitea_delete_runner "$repo" "$rid"; then
|
|
gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"repo\":\"$repo\",\"runner_id\":$rid,\"name\":\"$rname\"}"
|
|
else
|
|
gcr_log error --ns=sweep "failed deleting repo=$repo runner id=$rid"
|
|
fi
|
|
fi
|
|
done <<EOF
|
|
$runners
|
|
EOF
|
|
done <<EOF
|
|
$repos
|
|
EOF
|
|
IFS=,
|
|
done
|
|
IFS="$oldIFS"
|
|
gcr_lock_release idle-pool
|
|
}
|
|
|
|
# Runs SSH-push bootstrap for VMs that were created but not yet provisioned.
|
|
# Registration token is fetched fresh per attempt (short-lived usefulness).
|
|
gcr_bootstrap_pending() {
|
|
for f in $(gcr_active_records); do
|
|
rec="$(cat "$f")"
|
|
[ "$(gcr_record_field "$rec" status)" = "pending_vm" ] || continue
|
|
[ "$(gcr_record_field "$rec" bootstrapped)" = "true" ] && continue
|
|
|
|
job_id="$(gcr_record_field "$rec" job_id)"
|
|
attempt="$(gcr_record_field "$rec" run_attempt)"
|
|
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
|
gcr_lock_acquire "$key" || continue
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
if [ "$(gcr_record_field "$rec" status)" != "pending_vm" ] \
|
|
|| [ "$(gcr_record_field "$rec" bootstrapped)" = "true" ]; then
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
repo="$(gcr_record_field "$rec" repo)"
|
|
label="$(gcr_record_field "$rec" label)"
|
|
vm_id="$(gcr_record_field "$rec" vm_id)"
|
|
runner_name="$(gcr_record_field "$rec" vm_name)"
|
|
|
|
if [ "$(gcr_record_field "$rec" reused_vm)" = "true" ]; then
|
|
if gcr_vm_runner_service "$vm_id" start \
|
|
&& gcr_vm_runner_service "$vm_id" health \
|
|
&& gcr_gitea_runner_disabled "$repo" "$runner_name" false; then
|
|
rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | del(.reused_vm)')"
|
|
gcr_record_put "$job_id" "$attempt" "$rec"
|
|
else
|
|
gcr_gitea_runner_disabled "$repo" "$runner_name" true || true
|
|
fi
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
|
|
state="$(gcr_gitea_job_state "$repo" "$job_id")" || {
|
|
gcr_lock_release "$key"
|
|
continue
|
|
}
|
|
case "$state" in
|
|
completed:*)
|
|
gcr_log info --ns=sweep "pending job=$job_id already terminal ($state), destroying vm=$vm_id"
|
|
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
|
|
case "$state" in
|
|
completed:success|completed:cancelled|completed:skipped) ;;
|
|
*)
|
|
ip="$(gcr_vm_public_ip "$vm_id" || true)"
|
|
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
|
|
;;
|
|
esac
|
|
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
|
|
pending-job-completed false; then
|
|
gcr_event "vm-destroyed" "$job_id" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
|
|
else
|
|
gcr_event "vm-cleanup-pending" "$job_id" \
|
|
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\"}"
|
|
fi
|
|
else
|
|
gcr_record_del "$job_id" "$attempt"
|
|
fi
|
|
gcr_lock_release "$key"
|
|
continue
|
|
;;
|
|
esac
|
|
|
|
ip="$(gcr_vm_public_ip "$vm_id")" || {
|
|
gcr_lock_release "$key"
|
|
continue
|
|
}
|
|
if [ -z "$ip" ]; then
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
|
|
reg_token="$(gcr_gitea_registration_token "$repo")" || {
|
|
gcr_lock_release "$key"
|
|
continue
|
|
}
|
|
ttl_min="$(gcr_record_field "$rec" ttl_min)"
|
|
|
|
gcr_log info --ns=alloc "bootstrapping vm=$vm_id ip=$ip job=$job_id"
|
|
if gcr_vm_bootstrap_ssh "$ip" "$label" "$reg_token" "$runner_name"; then
|
|
rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | .ip = $ip' --arg ip "$ip")"
|
|
gcr_record_put "$job_id" "$attempt" "$rec"
|
|
gcr_event "vm-bootstrapped" "$job_id" "{\"vm_id\":$vm_id,\"ip\":\"$ip\"}"
|
|
else
|
|
gcr_log warn --ns=alloc "bootstrap failed vm=$vm_id (retry next tick)"
|
|
fi
|
|
gcr_lock_release "$key"
|
|
done
|
|
}
|
|
|
|
# Reap terminal jobs even when completed webhook path lags or is absent.
|
|
gcr_reap_finished_jobs() {
|
|
for f in $(gcr_active_records); do
|
|
rec="$(cat "$f")"
|
|
case "$(gcr_record_field "$rec" status)" in
|
|
pending_vm|vm_active) ;;
|
|
*) continue ;;
|
|
esac
|
|
|
|
job_id="$(gcr_record_field "$rec" job_id)"
|
|
attempt="$(gcr_record_field "$rec" run_attempt)"
|
|
repo="$(gcr_record_field "$rec" repo)"
|
|
vm_id="$(gcr_record_field "$rec" vm_id)"
|
|
|
|
state="$(gcr_gitea_job_state "$repo" "$job_id")" || continue
|
|
case "$state" in
|
|
completed:*)
|
|
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
|
gcr_lock_acquire "$key" || continue
|
|
rec="$(gcr_record_get "$job_id" "$attempt")"
|
|
if [ -z "$rec" ]; then
|
|
gcr_lock_release "$key"
|
|
continue
|
|
fi
|
|
case "$(gcr_record_field "$rec" status)" in
|
|
pending_vm|vm_active) ;;
|
|
*) gcr_lock_release "$key"; continue ;;
|
|
esac
|
|
finish_status=0
|
|
gcr_vm_finish_terminal "$job_id" "$attempt" "$rec" "$state" reconcile \
|
|
|| finish_status="$?"
|
|
gcr_lock_release "$key"
|
|
case "$finish_status" in
|
|
0|2) ;;
|
|
*) return "$finish_status" ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
gcr_tick() {
|
|
gcr_sweep_create_ambiguous
|
|
gcr_sweep_cleanup_pending
|
|
gcr_sweep_ttl
|
|
gcr_reap_finished_jobs
|
|
gcr_sweep_orphan_vms
|
|
gcr_retry_deferred
|
|
gcr_bootstrap_pending
|
|
gcr_sweep_stale_runners
|
|
}
|
|
|
|
gcr_main() {
|
|
: "${GCR_RECONCILE_INTERVAL_SEC:=60}"
|
|
gcr_state_init
|
|
|
|
gcr_log info --ns=core "controller starting, interval=${GCR_RECONCILE_INTERVAL_SEC}s state=$GCR_STATE_DIR"
|
|
gcr_tick
|
|
|
|
while :; do
|
|
sleep "$GCR_RECONCILE_INTERVAL_SEC"
|
|
if ! gcr_tick; then
|
|
gcr_log error --ns=core "tick failed, retrying next interval"
|
|
fi
|
|
done
|
|
}
|