Compare commits
34 Commits
0a2b96abc2
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
 yukkop
|
09eecf5969 | ||
|
yukkop
|
e444ea5936 | ||
|
yukkop
|
c917e4908c | ||
|
yukkop
|
ef849b085f | ||
|
yukkop
|
a09f247290 | ||
|
yukkop
|
3a52023082 | ||
|
yukkop
|
2bd466b652 | ||
|
yukkop
|
83cf9ff32f | ||
|
yukkop
|
a9e538fc76 | ||
|
yukkop
|
88ec8a59d8 | ||
|
yukkop
|
417820544e | ||
|
yukkop
|
1cba36e76e | ||
|
yukkop
|
4f97fb3244 | ||
|
yukkop
|
252d3418a5 | ||
|
yukkop
|
96882fd8cd | ||
|
yukkop
|
522010b7e6 | ||
|
yukkop
|
761120b211 | ||
|
yukkop
|
a04a58b792 | ||
|
yukkop
|
1e973e7b33 | ||
|
yukkop
|
66a502b1dd | ||
|
yukkop
|
fce9ae07be | ||
|
yukkop
|
a8c4a52a68 | ||
|
yukkop
|
078e01c87f | ||
|
yukkop
|
6bc564de59 | ||
|
yukkop
|
cc8a7cf80e | ||
|
yukkop
|
279df769db | ||
|
yukkop
|
069b18daa3 | ||
|
yukkop
|
37bd69e90e | ||
|
yukkop
|
572133a941 | ||
|
yukkop
|
ed721dd961 | ||
|
yukkop
|
4a88e7c6e8 | ||
|
yukkop
|
f0317efb6e | ||
|
yukkop
|
cb62bd3044 | ||
|
yukkop
|
ee8a33c3b0 |
@@ -98,13 +98,13 @@ jobs:
|
||||
true)
|
||||
nix run --refresh '.#with-attic-cache' -- -- \
|
||||
nix run --refresh '.#deploy' -- \
|
||||
push -- --flake '.#neuro|x86_64-linux' --target-host neuro --use-remote-sudo
|
||||
push -- --flake '.#neuro|x86_64-linux' --target-host neuro
|
||||
;;
|
||||
false)
|
||||
unset ATTIC_TOKEN
|
||||
timeout --kill-after=60s 21600s \
|
||||
nix run --refresh '.#deploy' -- \
|
||||
push -- --flake '.#neuro|x86_64-linux' --target-host neuro --use-remote-sudo
|
||||
push -- --flake '.#neuro|x86_64-linux' --target-host neuro
|
||||
;;
|
||||
*)
|
||||
printf 'unsupported upload_cache value: %s\n' "$UPLOAD_CACHE" >&2
|
||||
|
||||
+8
-1
@@ -6,12 +6,19 @@ keys:
|
||||
- &bfs-server age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
|
||||
- &bfs-pol-server age1fpytf05sg9n6ywpwkmn09lhpfvgtud9h75h76jhxha475zpnasqq952rpu
|
||||
- &bfs-new-server age17yx98qk9gzgcf2q6zhhp05p6mmtrkgz66dvyk9gqclypvlr8rersxjy5v7
|
||||
- &neuro-server age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
|
||||
- &neuro-server age1ak7heljpr0pjr4m0rcwxgn3sp0jjxw03lxyf33r8lcemqh2u2sgqx0aplq
|
||||
- &games-server age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
|
||||
- &hectic-lab-server age13h8twnwvgxn04l5ywtru89a6psw5d0uckr2eghxsjp88a5augvsstq5ard
|
||||
- &umbriel-bfs age1jxntjca8q2vxvf2jaal4xyvm2ae6sh62fhv897694kuzawfrk5asj00zdt
|
||||
|
||||
creation_rules:
|
||||
- path_regex: sus/neuro-minecraft.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- age1r25zdeqq8nac6dgca9en28r57ffyz9u9d8z5yc25gc8xqz747vaqmdtk0h
|
||||
- age1ak7heljpr0pjr4m0rcwxgn3sp0jjxw03lxyf33r8lcemqh2u2sgqx0aplq
|
||||
- age1ev53mzse6rg4ffwtcwtq4e93c7x7s4d0eyu89jrsahrke8r4yamsseu8h4
|
||||
|
||||
- path_regex: sus/home.xray.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
|
||||
@@ -224,18 +224,15 @@ Environment expected before real deploy/apply:
|
||||
S3 backend credentials and endpoint access
|
||||
a matching SOPS age identity for sus/gitea-runners.yaml
|
||||
kubectl access to the target cluster
|
||||
a concrete registry digest for the pushed Nix-capable runner image if enabling
|
||||
the nix label
|
||||
a valid Hetzner Nix image ID in the controller host configuration
|
||||
|
||||
OpenTofu validation gate:
|
||||
tofu version
|
||||
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||
|
||||
Nix image build/publish/digest gate:
|
||||
Nix image build/publish gate:
|
||||
nix build .#gitea-runner-nix-image
|
||||
publish the archive, then pin the registry-reported digest in the runner label
|
||||
mapping
|
||||
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||
publish/import image, then set nixImageId / GCR_NIX_IMAGE_ID to its Hetzner ID
|
||||
|
||||
SOPS token Secret creation gate:
|
||||
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||
@@ -272,7 +269,7 @@ Verification commands:
|
||||
Main blockers and gates:
|
||||
do not run tofu apply without all external inputs
|
||||
do not apply the k8s overlay until the gitea-runner-token Secret exists
|
||||
do not enable the nix label until the image has been published with a concrete digest
|
||||
do not dispatch nix jobs until nixImageId / GCR_NIX_IMAGE_ID is valid
|
||||
do not print, load, or require secrets on shell entry
|
||||
EOF
|
||||
'';
|
||||
|
||||
+11
-5
@@ -1,13 +1,15 @@
|
||||
# Documentation
|
||||
|
||||
- [Using the `hectic` Attic Cache](./attic-cache.md)
|
||||
- [Minecraft incident log](./minecraft-incidents.md)
|
||||
- [Project Zomboid backups](./project-zomboid-backups.md)
|
||||
|
||||
## Gitea runner labels
|
||||
|
||||
Common labels for zero-idle runners:
|
||||
Common labels for controller-managed on-demand runners:
|
||||
|
||||
- `ubuntu-latest` — persistent Ubuntu 24.04 `cx23` worker
|
||||
- `nix` — persistent Nix-capable Ubuntu `cx23` worker
|
||||
- `ubuntu-latest` — on-demand alias for `gross-x86`
|
||||
- `nix` — on-demand Nix alias with 480-minute TTL
|
||||
- `gross-x86` — x86 fallback chain `cx53` / `cx43` / `cx33`
|
||||
- `gross-arm` — ARM fallback chain `cax41` / `cax31` / `cax21`
|
||||
- `gross-x86-perf` — x86 performance chain `cx53` / `cpx62` / `cpx52`
|
||||
@@ -15,12 +17,16 @@ Common labels for zero-idle runners:
|
||||
- `gross-nix-x86` — `gross-x86` + Nix bootstrap
|
||||
- `gross-nix-arm` — `gross-arm` + Nix bootstrap
|
||||
- `gross-nix-x86-perf` — `gross-x86-perf` + Nix bootstrap
|
||||
- `gross-nix-x86-highmem` — CCX53-only Nix runner, 480-minute TTL
|
||||
- `gross-nix-mixed-econ` — `gross-mixed-econ` + Nix bootstrap
|
||||
|
||||
Region order for fallback: `nbg1`, then `fsn1`, then `hel1`.
|
||||
|
||||
`nix` and `ubuntu-latest` are persistent workers; only `gross-*` labels use
|
||||
zero-idle ephemeral VMs.
|
||||
The legacy Kubernetes persistent pool is disabled (`replicas: 0`) and has no
|
||||
registered labels. All listed labels are handled by the VM controller. After a
|
||||
successful job, a bootstrapped VM remains running and idle until its next hourly
|
||||
lifetime boundary, capped by label TTL. A queued job from same repository with
|
||||
same label reuses it without another VM creation or budget reservation.
|
||||
|
||||
Operational details: `infra/gitea-runners/runbook.md` and
|
||||
`package/gitea-runner-controller/decide.sh`.
|
||||
|
||||
+3
-3
@@ -201,9 +201,9 @@ build/deploy command, batches of at most 8 paths, and 600 seconds per upload att
|
||||
upload deadline covers the **whole batch**, not each individual path. Its final
|
||||
drain is bounded at 1 hour; the 435-minute job budget leaves 15 minutes for setup
|
||||
and cleanup. The `gross-nix-x86-perf` runner limit and Gitea's endless-task
|
||||
watchdog are 8 hours. The VM lifetime starts at allocation and includes the
|
||||
controller's additional 10-minute grace. A prolonged cache outage can still
|
||||
exhaust the drain before every queued path is uploaded.
|
||||
watchdog are 8 hours. VM hard lifetime starts at allocation and has no controller
|
||||
destruction grace. A prolonged cache outage can still exhaust the drain before
|
||||
every queued path is uploaded.
|
||||
|
||||
The build timeout covers the entire wrapped command, not each derivation.
|
||||
Completed outputs can be reused from the cache, but an interrupted CUDA/Magma
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
# Minecraft incident log
|
||||
|
||||
This file records only observed evidence, actions, and verification results.
|
||||
An entity appearing in a stack trace is a trigger-path observation, not a
|
||||
proven root cause.
|
||||
|
||||
## 2026-09-19 — WorldOfSosal crashes in Sable block-change handling
|
||||
|
||||
### Impact
|
||||
|
||||
- `minecraft-server-wowMineMap.service` terminates while a player is online.
|
||||
- Public Minecraft endpoint is `store.hectic-lab.com:25568`.
|
||||
- Server is intentionally stopped after the latest crash to prevent repeated
|
||||
crash-save cycles while recovery is investigated.
|
||||
|
||||
### Observed evidence
|
||||
|
||||
All crash reports contain `sable@2.0.5` in
|
||||
`LevelAccelerator.getBlockState`, followed by
|
||||
`ArrayIndexOutOfBoundsException` where the requested section index exceeds
|
||||
the world section array length of `24`.
|
||||
|
||||
| UTC timestamp | Crash report | Observed trigger path | Exception |
|
||||
| --- | --- | --- | --- |
|
||||
| 18:47:03 | `crash-2026-09-19_18.47.03-server.txt` | `EnderMan$EndermanTakeBlockGoal.tick` | index `38` / length `24` |
|
||||
| 18:52:17 | `crash-2026-09-19_18.52.17-server.txt` | `GlowSquid.aiStep` → `RedStoneOreBlock.stepOn` | index `33` / length `24` |
|
||||
| 19:14:46 | `crash-2026-09-19_19.14.46-server.txt` | `Skeleton.tick` → `RedStoneOreBlock.stepOn` | index `34` / length `24` |
|
||||
|
||||
Evidence locations on `neuro`:
|
||||
|
||||
```text
|
||||
/srv/minecraft/wowMineMap/crash-reports/
|
||||
/srv/minecraft/wowMineMap/logs/latest.log
|
||||
```
|
||||
|
||||
### Actions performed
|
||||
|
||||
| UTC timestamp | Action | Result |
|
||||
| --- | --- | --- |
|
||||
| 17:51 | Archived current world before recovery | Archive checksum recorded |
|
||||
| 18:08 | Set `randomTickSpeed=0` | Server started, but later crashed from an entity block change |
|
||||
| 18:48 | Set `mobGriefing=false` | Prevented Enderman block pickup only; later crashes still occurred |
|
||||
| 18:54 | Archived post-crash world | Archive checksum recorded |
|
||||
| 19:00 | Moved Boss offline player NBT from `(3299.067, 142.630, 8613.742)` to `(3296, 500, 8608)` in `crafting_azeroth:azeroth` | Only `Pos` and `Dimension` changed; later crash still occurred |
|
||||
| after 19:14 crash | Stopped `minecraft-server-wowMineMap.service` | Prevented further automatic crash/restart saves |
|
||||
|
||||
### Recovery artifacts
|
||||
|
||||
```text
|
||||
/srv/minecraft/backups/wowMineMap-before-sable-recovery-20260919T175139Z.tar.zst
|
||||
/srv/minecraft/backups/wowMineMap-after-sable-crashes-20260919T185445Z.tar.zst
|
||||
/srv/minecraft/wowMineMap/world/playerdata/1c189af5-2713-3fa6-bcc4-893dfadedfa4.dat.before-relocation
|
||||
```
|
||||
|
||||
### Conclusions supported by evidence
|
||||
|
||||
- Public proxy and reverse tunnel are not the failure point: server-list ping
|
||||
succeeded before later in-world crashes.
|
||||
- The failure is not limited to Endermen, random ticks, or one player
|
||||
position.
|
||||
- Sable's block-change callback is present in every captured crash.
|
||||
|
||||
### Not established
|
||||
|
||||
- Exact corrupt chunk, block, or mod data.
|
||||
- Whether world data is corrupt, Sable itself is defective, or another mod is
|
||||
supplying incompatible world state.
|
||||
- Whether deleting any chunk, region, or Sable state would be safe.
|
||||
|
||||
### External research
|
||||
|
||||
No exact upstream match was found for Sable `2.0.5` on NeoForge `1.21.1` with
|
||||
`LevelAccelerator.getBlockState` and a requested section index of `33`, `34`,
|
||||
or `38` against a section array of length `24`.
|
||||
|
||||
Related but non-identical upstream reports:
|
||||
|
||||
- [Sable #776](https://github.com/ryanhcode/sable/issues/776) documents an
|
||||
`ArrayIndexOutOfBoundsException` associated with unusual dimension height
|
||||
bounds. This is relevant to section-coordinate handling, but is an older
|
||||
version and different stack trace.
|
||||
- [Sable #1087](https://github.com/ryanhcode/sable/issues/1087) documents a
|
||||
`LevelAccelerator.getBlockState` recursion during block-shape processing.
|
||||
The failure type differs.
|
||||
- [Sable #820](https://github.com/ryanhcode/sable/issues/820) documents a
|
||||
ticking-entity block-change crash. The reported downgrade to `1.1.3` helped
|
||||
that distinct recursive-update failure; it is not evidence for this crash.
|
||||
- [Sable #1223](https://github.com/ryanhcode/sable/issues/1223) documents a
|
||||
different `ArrayIndexOutOfBoundsException` in voxel-neighborhood handling.
|
||||
Its suggested Lithium setting only reduced crashes for some reporters and is
|
||||
not a verified mitigation here.
|
||||
|
||||
Sable `2.0.4` and `2.0.5` release notes mention other block or contraption
|
||||
crash fixes, but not this exception. No version upgrade or downgrade is
|
||||
currently evidence-backed as a production fix.
|
||||
|
||||
### Next recovery step
|
||||
|
||||
Use a disposable full-world copy to test a supported Sable/physics integration
|
||||
mitigation. Do not restart production, delete region files, or overwrite a
|
||||
backup until that test gives reproducible evidence.
|
||||
Executable
+231
@@ -0,0 +1,231 @@
|
||||
# WorldOfSosal: Prism automatic updates
|
||||
|
||||
The published client entry points are:
|
||||
- https://store.bfs.band/minecraft/ (BFS / Element host)
|
||||
- https://store.hectic-lab.com/minecraft/world-of-sosal/ (hectic-lab)
|
||||
|
||||
Each site provides its own Prism ZIP with that site's update URL and matching
|
||||
server address. Both installs use the same Minecraft world and modpack release.
|
||||
|
||||
Players import `WorldOfSosal-Prism.zip` into Prism once and approve its pre-launch
|
||||
command. Before each launch, packwiz-installer reconciles the client with the
|
||||
published pack: it adds, replaces, and removes managed files, checking hashes.
|
||||
`options.txt` is seeded once and preserved. Pack configuration files are managed
|
||||
and can be replaced. Upstream mods do not update independently of your release.
|
||||
Minecraft 1.21.1, NeoForge 21.1.250, Java 21; the instance reserves up to 8 GiB.
|
||||
|
||||
The original `.mrpack` alone does not provide this automatic update mechanism.
|
||||
Official workflow: https://packwiz.infra.link/tutorials/installing/packwiz-installer/
|
||||
|
||||
## Publishing a tested update
|
||||
|
||||
Keep the authoritative `.mrpack` in Storage Box at
|
||||
`minecraft/pack/WorldOfSosal.mrpack`. For a server update, replace that archive,
|
||||
set its new SHA-256 in `nixos/system/neuro/minecraft/world-of-sosal.nix`,
|
||||
and rebuild/switch neuro before publishing the corresponding client export. The server importer and the
|
||||
client export must consume the same archive; publishing only the client can make
|
||||
it incompatible with the running server.
|
||||
|
||||
```sh
|
||||
# Test the client and deploy the matching server release first.
|
||||
python3 script/publish-prism-mirrors.py WorldOfSosal.mrpack
|
||||
```
|
||||
|
||||
The mirror publisher creates temporary build directories and sets each server
|
||||
address and update URL automatically. The builder downloads a SHA-256-pinned bootstrap from the
|
||||
packwiz project's release, or accepts it via `--bootstrap /path/to/file.jar`.
|
||||
External mods retain their original URLs and SHA-512 checksums. Embedded mods and
|
||||
configuration are hosted with the release. Both required and optional client mods
|
||||
are included, matching the current server importer's optional-mod behavior.
|
||||
|
||||
Publishing uploads an immutable directory, checks it if it already exists, and
|
||||
atomically switches `current`. Previous directories remain available for rollback.
|
||||
Do not remove a release while clients may still be reading it. Hash checks cause
|
||||
an overlapping update to fail safely rather than silently accept mixed contents;
|
||||
retry the launch if a publication overlapped a download.
|
||||
|
||||
The files live under `/var/www/store/minecraft/world-of-sosal` on `hectic-lab`,
|
||||
served by the existing `store.hectic-lab.com` nginx virtual host. No nginx reload
|
||||
is needed for pack updates. Keep `current/pack.toml` as the stable client URL.
|
||||
The index must be alongside pack.toml: putting a release prefix in `[index].file`
|
||||
also prefixes client installation paths with that directory in packwiz-installer.
|
||||
|
||||
If Minecraft/NeoForge versions change, update and test both the server pin and
|
||||
client pack. packwiz-installer 0.5.14 understands NeoForge components in Prism's
|
||||
`mmc-pack.json`; a launcher restart/relaunch may be necessary after changing them.
|
||||
|
||||
## Verification on 2026-09-18
|
||||
|
||||
- Source archive SHA-256:
|
||||
`f8c18acb9208e4592725632ae50dab4f9c308483b34fd43a6507c74fdbf8169f`.
|
||||
- Public HTTPS installation into a clean Prism-format instance passed: all 141
|
||||
client mods and all overrides match the original archive. A second launch
|
||||
performed no downloads and preserved personal options.
|
||||
- Direct probes of neuro public ports 25565, 25567, and 25568 timed out;
|
||||
the configured relay now provides the public entry point.
|
||||
- Live WoW server reached `Done` with all 135 server mod SHA-512 hashes
|
||||
matching the same archive used for the Prism client.
|
||||
- Public `store.hectic-lab.com:25568` status/ping succeeded (about 111 ms);
|
||||
a login handshake reached the online authentication encryption request.
|
||||
An authenticated Windows Prism session was subsequently verified on 2026-09-19 (see below).
|
||||
- Server and tunnel are enabled at boot; relay and both NixOS configurations
|
||||
are deployed. No failed systemd units remain on neuro.
|
||||
- Loader package `neoforge-1.21.1-21.1.250` built successfully in Nix.
|
||||
- Automatic updater add/remove/config-update and options-preservation behavior
|
||||
tested with an actual packwiz-installer run against a controlled update fixture.
|
||||
|
||||
## WoW server and public entry point
|
||||
|
||||
The WoW map and WorldOfSosal mods share the `wowMineMap` server on neuro,
|
||||
listening on 25567. There is no separate WorldOfSosal world/server on 25568.
|
||||
The client pack and server both pin Minecraft 1.21.1 / NeoForge 21.1.250.
|
||||
Map import runs before mod import, and both finish before Minecraft starts.
|
||||
|
||||
The public entry point is `store.hectic-lab.com:25568`:
|
||||
|
||||
```
|
||||
Prism -> hectic-lab:25568 -> loopback:25577 -> SSH tunnel -> neuro:25567
|
||||
```
|
||||
|
||||
`minecraft-wow-proxy.socket` and its socket-proxyd service run on hectic-lab.
|
||||
`minecraft-wow-tunnel.service` on neuro establishes a reverse SSH forward and
|
||||
reconnects after failures. A dedicated SSH identity may listen only on
|
||||
127.0.0.1:25577 at the relay; it has no interactive shell or other forwarding.
|
||||
Both services and firewall rules are in Nix and start on boot. The SSH client
|
||||
uses an explicit AES-CTR / HMAC-SHA256-ETM / curve25519 transport profile with
|
||||
IPQoS=none, tested on the neuro-to-lab route. The default profile stalled after
|
||||
the handshake on this route. Both ends check peer liveness so stale listeners
|
||||
are eventually released. Minecraft initially used `online-mode=true`. It now uses offline mode at the
|
||||
owner's request; see the RCON and authentication section below.
|
||||
|
||||
For a temporary direct local tunnel, use:
|
||||
|
||||
```sh
|
||||
ssh -NTL 0.0.0.0:25568:127.0.0.1:25567 \
|
||||
-o ExitOnForwardFailure=yes \
|
||||
-o ServerAliveInterval=30 -o ServerAliveCountMax=3 neuro
|
||||
```
|
||||
|
||||
That command exposes the local 25568 listener on all interfaces, as requested.
|
||||
Use 127.0.0.1 instead of the first 0.0.0.0 if only this computer should use it.
|
||||
|
||||
Credentials are encrypted in `sus/neuro-minecraft.yaml` with the actual neuro
|
||||
host identity and owner keys. The existing `sus/neuro.yaml` is unchanged.
|
||||
The source WoW archive remains untouched in Storage Box. Import is idempotent:
|
||||
an existing world with level.dat is preserved. Never delete the world to update
|
||||
mods; publish/deploy a matching modpack release instead.
|
||||
|
||||
Useful checks:
|
||||
|
||||
```sh
|
||||
ssh neuro systemctl status minecraft-world-import-wowMineMap \
|
||||
minecraft-modpack-import-worldOfSosal minecraft-server-wowMineMap \
|
||||
minecraft-wow-tunnel --no-pager
|
||||
ssh hectic-lab systemctl status minecraft-wow-proxy.socket --no-pager
|
||||
ssh neuro journalctl -u minecraft-server-wowMineMap -n 80 --no-pager
|
||||
```
|
||||
|
||||
The initial isolated server compatibility test reached `Done` and answered the
|
||||
Minecraft status/ping protocol. Its logs also contain nonfatal recipe and class
|
||||
function errors from the supplied modpack; successful startup does not imply that
|
||||
every recipe or RPG class feature works correctly.
|
||||
|
||||
The imported map metadata is `wow mine`, DataVersion 3953 (Minecraft 1.21),
|
||||
spawn 0 / 68 / -32; extracted size is approximately 11.7 GiB. The archive
|
||||
SHA-256 was verified before extraction.
|
||||
|
||||
## Windows Prism GUI verification on 2026-09-19
|
||||
|
||||
- Downloaded the published ZIP through the browser and imported it in Prism 8.4.
|
||||
- Fixed the generated instance.cfg: ConfigVersion=1.2 is required. Without it,
|
||||
Prism selects its legacy INI parser and corrupts the quoted pre-launch command.
|
||||
The corrected ZIP is published at the same URL. Previously imported copies
|
||||
need the command corrected in Settings / Custom commands, or a fresh import.
|
||||
- Used Java 21.0.4; the first packwiz download hit two transient timeouts.
|
||||
Cancelled the incomplete launch and retried successfully. All 141 downloaded
|
||||
client mod hashes match the original mrpack. NeoForge reports 202 mods when
|
||||
bundled/internal mod components are included.
|
||||
- Joined store.hectic-lab.com:25568 in the actual Minecraft GUI. The server
|
||||
confirmed the authenticated join, and the client reached the Origins selection
|
||||
screen. No character origin was selected during testing.
|
||||
- Tested a separate copy of the pack manifest with an inert config text file:
|
||||
launching from Prism added it; restoring the production manifest and launching
|
||||
again automatically deleted it. Existing files were reused from cache, and
|
||||
options.txt retained its checksum. The production pack contents were unchanged.
|
||||
- Restored the instance's regular current/pack.toml update URL.
|
||||
|
||||
## Independent BFS entry point (2026-09-19)
|
||||
|
||||
- Server: `wow.bfs.band`; downloads: https://store.bfs.band/minecraft/.
|
||||
- BFS is `bfs.poland.xray` (91.198.166.181), the host of Element.
|
||||
- `minecraft-wow-tunnel-bfs` connects neuro directly to BFS. The BFS path does
|
||||
not transit hectic-lab; both tunnels have independent reconnecting services.
|
||||
- Shared proxy implementation: `nixos/module/generic/minecraft-public-relay.nix`.
|
||||
Host settings remain in `minecraft-wow-proxy.nix` (hectic-lab) and
|
||||
`minecraft-wow.nix` (BFS). A dedicated HTTPS virtual host serves `store.bfs.band`. The legacy
|
||||
`bfs.band/minecraft/` URLs remain available for already imported instances.
|
||||
- Downloaded BFS ZIP seeds `wow.bfs.band` and uses the stable manifest
|
||||
`https://store.bfs.band/minecraft/world-of-sosal/current/pack.toml`. It does not
|
||||
redirect installation metadata to hectic-lab. Upstream mod and Java/loader
|
||||
downloads still use their original providers (e.g. Modrinth, GitHub, Mojang).
|
||||
- Existing hectic-lab instances can be migrated without reinstalling mods:
|
||||
in Edit / Settings / Custom commands, replace only the manifest URL in
|
||||
Pre-launch command with the BFS URL above. Change the multiplayer server
|
||||
address to wow.bfs.band. New users should import the ZIP from BFS.
|
||||
- `script/publish-prism-mirrors.py` builds host-specific ZIPs from one archive
|
||||
and publishes both mirrors. It checks that the running neuro server's cached
|
||||
archive has the same SHA-256. Each host's switch is atomic; publication across
|
||||
two hosts is sequential, so rerun the command if it exits unsuccessfully.
|
||||
- Both configurations were deployed; public Minecraft status/ping succeeds
|
||||
on BFS (~125 ms), HTTPS serves the pack, and Element/Matrix HTTP checks pass.
|
||||
|
||||
Clean installation through the BFS manifest passed: all 141 client mods and
|
||||
all overrides match the source archive. A second updater run performed no
|
||||
downloads and preserved options.txt. The public BFS login protocol reached
|
||||
online authentication; the earlier full GUI login used hectic-lab.
|
||||
|
||||
## BFS DNS and dedicated download site (2026-09-19)
|
||||
|
||||
Porkbun DNS, TTL 600:
|
||||
|
||||
| Type | Name | Value |
|
||||
| --- | --- | --- |
|
||||
| A | store.bfs.band | 91.198.166.181 |
|
||||
| A | wow.bfs.band | 91.198.166.181 |
|
||||
| SRV | _minecraft._tcp.wow.bfs.band | 0 0 25568 wow.bfs.band |
|
||||
|
||||
Players enter `wow.bfs.band` without a port in Minecraft Java. In Porkbun,
|
||||
SRV Priority is `0`, and Target is `0 25568 wow.bfs.band` (weight, port, host).
|
||||
The root download URL https://store.bfs.band/ redirects to the WorldOfSosal page.
|
||||
The NixOS virtual host obtains and renews its HTTPS certificate automatically.
|
||||
The publication script now seeds this update URL and the port-free game address.
|
||||
Existing BFS instances retain working legacy update URLs; switching their
|
||||
pre-launch manifest to the new store host is optional. Root bfs.band remains
|
||||
the existing Element entry point.
|
||||
|
||||
## RCON and authentication (2026-09-19)
|
||||
|
||||
The WoW server now has `online-mode=false`. Account authentication is disabled;
|
||||
player names can be impersonated, and offline UUIDs differ from online UUIDs.
|
||||
Existing inventory/permissions may require a separate UUID migration.
|
||||
|
||||
RCON listens on TCP 25575 on neuro; its port is not opened in the firewall or
|
||||
forwarded through the public Minecraft relays. The server-specific automatic
|
||||
firewall is disabled and only game port 25567 is explicitly permitted.
|
||||
A random password is stored in SOPS as `minecraft/rcon-password`, injected into
|
||||
server.properties at startup with mode 0600, and is absent from the Nix store.
|
||||
|
||||
Start a local-only SSH tunnel and leave it running:
|
||||
|
||||
```sh
|
||||
ssh -NT -L 127.0.0.1:25575:127.0.0.1:25575 -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 neuro
|
||||
```
|
||||
|
||||
Retrieve the password in another terminal (do not paste it into logs):
|
||||
|
||||
```sh
|
||||
ssh neuro cat /run/secrets/minecraft/rcon-password
|
||||
```
|
||||
|
||||
Configure the RCON client with host `127.0.0.1`, port `25575`, and that password.
|
||||
There is no RCON username. These changes apply to wowMineMap only.
|
||||
@@ -0,0 +1,96 @@
|
||||
# Project Zomboid backups
|
||||
|
||||
`hectic.services."project-zomboid".backup` creates local backups without stopping
|
||||
or pausing the server. The default schedule is every 30 minutes. Each run:
|
||||
|
||||
1. sends the local RCON `save` command and waits for the configured save grace
|
||||
period;
|
||||
2. rsyncs `Zomboid/Saves/Multiplayer/<serverName>` and non-secret server
|
||||
settings (`SandboxVars`, spawn-points, and spawn-regions) from
|
||||
`Zomboid/Server` into a private staging tree;
|
||||
3. waits five seconds and repeats the rsync to narrow the live-write window;
|
||||
4. publishes a timestamped `tar.zst` archive; and
|
||||
5. deletes local archives older than `backup.retentionDays`.
|
||||
|
||||
The service lock prevents overlapping runs. Missing save or server-config paths
|
||||
skip the run through systemd `ConditionPathExists` checks.
|
||||
|
||||
## Consistency and secrets
|
||||
|
||||
This is a best-effort backup. It does not stop Project Zomboid and does not use
|
||||
an atomic filesystem snapshot. The RCON save command flushes the world before
|
||||
copying, and the second rsync narrows the remaining live-write window, but
|
||||
neither makes the filesystem copy an atomic snapshot.
|
||||
|
||||
Archives do not include the generated server INI, `admin-password`,
|
||||
host-generated password files, or the S3 credentials file. The server INI is
|
||||
generated again during service startup; provision secret-backed values separately
|
||||
after a restore.
|
||||
|
||||
## hectic-lab
|
||||
|
||||
hectic-lab runs the timer every 30 minutes and keeps local archives for 14 days:
|
||||
|
||||
```text
|
||||
/var/lib/project-zomboid/backups/archive/
|
||||
```
|
||||
|
||||
Check it with:
|
||||
|
||||
```sh
|
||||
systemctl list-timers project-zomboid-backup.timer
|
||||
systemctl status project-zomboid-backup.service
|
||||
journalctl -u project-zomboid-backup.service
|
||||
```
|
||||
|
||||
RCON is enabled on localhost port `27015`; the firewall does not expose this
|
||||
port. The password is generated at
|
||||
`/var/lib/project-zomboid/rcon-password` with mode `0600`. The server also uses
|
||||
`SaveWorldEveryMinutes=15` as a periodic persistence fallback.
|
||||
|
||||
## Optional S3 upload
|
||||
|
||||
S3 upload is disabled by default. Enabling it requires `bucket`, `endpoint`,
|
||||
`region`, and an absolute runtime `credentialsFile` outside `/nix/store`. The
|
||||
endpoint must use HTTPS. systemd reads the environment file without executing
|
||||
it; this host keeps it owned by `project-zomboid` with mode `0400`:
|
||||
|
||||
```sh
|
||||
AWS_ACCESS_KEY_ID=...
|
||||
AWS_SECRET_ACCESS_KEY=...
|
||||
```
|
||||
|
||||
Set `backup.s3.prefix` to choose the object-key prefix and
|
||||
`backup.s3.remoteRetentionDays` to prune old archives from that prefix. Remote
|
||||
deletion runs only after a successful upload and only matches this server's
|
||||
archive name prefix. Configure bucket lifecycle expiration/versioning too when
|
||||
available; it remains the stronger recovery and cleanup control.
|
||||
|
||||
## Restore
|
||||
|
||||
Restoring must be done while the server is stopped so it cannot modify files
|
||||
during extraction:
|
||||
|
||||
The versioned helper creates a fresh current-state backup, stops the timer and
|
||||
server, validates archive paths, restores the save, and starts both services:
|
||||
|
||||
```sh
|
||||
sudo ./docs/project-zomboid-restore.sh \
|
||||
/var/lib/project-zomboid/backups/archive/<archive>.tar.zst
|
||||
```
|
||||
|
||||
It writes a rollback archive named
|
||||
`project-zomboid-<serverName>-pre-restore-<timestamp>.tar.zst` before changing
|
||||
the save.
|
||||
|
||||
```sh
|
||||
systemctl stop project-zomboid.service
|
||||
tar --zstd --no-same-owner --no-same-permissions \
|
||||
-xf /var/lib/project-zomboid/backups/archive/<archive>.tar.zst \
|
||||
-C /var/lib/project-zomboid
|
||||
chown -R project-zomboid:project-zomboid /var/lib/project-zomboid/Zomboid
|
||||
systemctl start project-zomboid.service
|
||||
```
|
||||
|
||||
Re-provision password files and secret-backed INI values before starting.
|
||||
Verify the restored save and server name before allowing players to reconnect.
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SERVER_NAME=${SERVER_NAME:-servertest}
|
||||
DATA_DIR=${DATA_DIR:-/var/lib/project-zomboid}
|
||||
ARCHIVE=${1:-}
|
||||
|
||||
usage() {
|
||||
printf '%s\n' "Usage: $0 /path/to/project-zomboid-${SERVER_NAME}-<timestamp>.tar.zst"
|
||||
printf '%s\n' "Environment: SERVER_NAME, DATA_DIR"
|
||||
}
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
printf '%s\n' 'Run as root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$ARCHIVE" ]; then
|
||||
usage >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ ! -r "$ARCHIVE" ]; then
|
||||
printf 'Backup archive is not readable: %s\n' "$ARCHIVE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE_DIR="$DATA_DIR/backups/archive"
|
||||
SAVE_DIR="$DATA_DIR/Zomboid/Saves/Multiplayer/$SERVER_NAME"
|
||||
SERVER_DIR="$DATA_DIR/Zomboid/Server"
|
||||
TMP_LIST=$(mktemp)
|
||||
ROLLBACK_ARCHIVE=''
|
||||
SERVER_STOPPED=0
|
||||
RESTORE_SUCCEEDED=0
|
||||
|
||||
cleanup() {
|
||||
rm -f "$TMP_LIST"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
status=$?
|
||||
if [ "$status" -ne 0 ] && [ "$SERVER_STOPPED" -eq 1 ] \
|
||||
&& [ "$RESTORE_SUCCEEDED" -eq 0 ] && [ -n "$ROLLBACK_ARCHIVE" ]; then
|
||||
set +e
|
||||
rm -rf "$SAVE_DIR"
|
||||
rm -f \
|
||||
"$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \
|
||||
"$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \
|
||||
"$SERVER_DIR/${SERVER_NAME}_spawnregions.lua"
|
||||
tar --zstd --no-same-owner --no-same-permissions \
|
||||
-xpf "$ROLLBACK_ARCHIVE" -C "$DATA_DIR"
|
||||
chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR"
|
||||
systemctl start project-zomboid.service
|
||||
systemctl start project-zomboid-backup.timer
|
||||
printf '%s\n' "Restore failed; current state restored from $ROLLBACK_ARCHIVE" >&2
|
||||
fi
|
||||
cleanup
|
||||
exit "$status"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
if ! tar --zstd -tf "$ARCHIVE" >"$TMP_LIST"; then
|
||||
printf 'Archive integrity check failed: %s\n' "$ARCHIVE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while IFS= read -r member; do
|
||||
case "$member" in
|
||||
Zomboid/*) ;;
|
||||
*)
|
||||
printf 'Unsafe archive member: %s\n' "$member" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
case "$member" in
|
||||
/*|*../*)
|
||||
printf 'Path traversal member: %s\n' "$member" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done <"$TMP_LIST"
|
||||
|
||||
if ! systemctl start project-zomboid-backup.service; then
|
||||
printf '%s\n' 'Could not create fresh backup of current state.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CURRENT_ARCHIVE=$(find "$ARCHIVE_DIR" -maxdepth 1 -type f \
|
||||
-name "project-zomboid-$SERVER_NAME-*.tar.zst" \
|
||||
-printf '%T@ %p\n' | sort -nr | awk 'NR == 1 {sub(/^[^ ]* /, ""); print}')
|
||||
|
||||
if [ -z "$CURRENT_ARCHIVE" ]; then
|
||||
printf '%s\n' 'Fresh current-state backup was not found.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
stamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
ROLLBACK_ARCHIVE="$ARCHIVE_DIR/project-zomboid-$SERVER_NAME-pre-restore-$stamp.tar.zst"
|
||||
cp --reflink=auto "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE" 2>/dev/null \
|
||||
|| cp "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE"
|
||||
chmod 0600 "$ROLLBACK_ARCHIVE"
|
||||
chown project-zomboid:project-zomboid "$ROLLBACK_ARCHIVE"
|
||||
|
||||
systemctl stop project-zomboid-backup.timer
|
||||
systemctl stop project-zomboid.service
|
||||
SERVER_STOPPED=1
|
||||
|
||||
if [ "$(systemctl show project-zomboid --property=ActiveState --value)" != inactive ]; then
|
||||
printf '%s\n' 'Project Zomboid did not stop; refusing to restore.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -rf "$SAVE_DIR"
|
||||
rm -f \
|
||||
"$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \
|
||||
"$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \
|
||||
"$SERVER_DIR/${SERVER_NAME}_spawnregions.lua"
|
||||
|
||||
tar --zstd --no-same-owner --no-same-permissions \
|
||||
-xpf "$ARCHIVE" -C "$DATA_DIR"
|
||||
chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR"
|
||||
|
||||
systemctl start project-zomboid.service
|
||||
started=0
|
||||
for _ in $(seq 1 90); do
|
||||
if [ "$(systemctl show project-zomboid --property=ActiveState --value)" = active ] \
|
||||
&& [ "$(systemctl show project-zomboid --property=SubState --value)" = running ]; then
|
||||
started=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "$started" -ne 1 ]; then
|
||||
printf 'Restore completed, but service did not become healthy. Rollback: %s\n' \
|
||||
"$ROLLBACK_ARCHIVE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
systemctl start project-zomboid-backup.timer
|
||||
RESTORE_SUCCEEDED=1
|
||||
printf 'Restore completed.\n'
|
||||
printf 'Rollback archive: %s\n' "$ROLLBACK_ARCHIVE"
|
||||
Generated
+45
-1
@@ -625,6 +625,28 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"iana-angl": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1789498292,
|
||||
"narHash": "sha256-bsrT7MWTXL+tpwDZmI5xWLPZZPYAzU1WjcGlNmqpePw=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "2937d257d601b40de2437a51ebc3c7b61b40f679",
|
||||
"revCount": 37,
|
||||
"type": "git",
|
||||
"url": "https://gitea.hectic-lab.com/yukkop/learning.git"
|
||||
},
|
||||
"original": {
|
||||
"rev": "2937d257d601b40de2437a51ebc3c7b61b40f679",
|
||||
"type": "git",
|
||||
"url": "https://gitea.hectic-lab.com/yukkop/learning.git"
|
||||
}
|
||||
},
|
||||
"impermanence": {
|
||||
"locked": {
|
||||
"lastModified": 1737831083,
|
||||
@@ -986,6 +1008,7 @@
|
||||
"hectic-landing": "hectic-landing",
|
||||
"home-manager": "home-manager",
|
||||
"hyprland": "hyprland",
|
||||
"iana-angl": "iana-angl",
|
||||
"impermanence": "impermanence",
|
||||
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
|
||||
"nix-darwin": "nix-darwin",
|
||||
@@ -996,11 +1019,32 @@
|
||||
"nixos-wsl": "nixos-wsl",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"nixvim": "nixvim",
|
||||
"rust-overlay": "rust-overlay",
|
||||
"rust-overlay": "rust-overlay_2",
|
||||
"sops-nix": "sops-nix"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"iana-angl",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1789457514,
|
||||
"narHash": "sha256-Aggle++fTyAifBy+QBPxjM+obO5iepKW/8MDxQtgGvI=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "89e99bf0778a8f2cd18c9360c3f19c1ee47fc739",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"rust-overlay_2": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
|
||||
@@ -72,6 +72,10 @@
|
||||
url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
iana-angl = {
|
||||
url = "git+https://gitea.hectic-lab.com/yukkop/learning.git?rev=2937d257d601b40de2437a51ebc3c7b61b40f679";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
mechabellum-replay-analysis = {
|
||||
# NOTE(yukkop): private repo - SSH access required.
|
||||
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
||||
|
||||
@@ -24,43 +24,30 @@ Preferred registry:
|
||||
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
|
||||
```
|
||||
|
||||
Publish the archive without adding secrets to the image layers, then use the
|
||||
registry-reported digest as the only final `nix` label image reference:
|
||||
Publish the archive without adding secrets to the image layers. Controller-owned
|
||||
zero-idle runners select this image through `nixImageId` / `GCR_NIX_IMAGE_ID`;
|
||||
they do not use a Gitea label-to-container-image mapping:
|
||||
|
||||
```text
|
||||
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||
nixImageId = "<Hetzner-image-id>";
|
||||
```
|
||||
|
||||
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
|
||||
runner label mapping must use the `@sha256:` reference above. Keep
|
||||
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
|
||||
configuration task explicitly changes it. Only the `nix` label should select
|
||||
this custom image.
|
||||
The `2026-06-07` tag may be pushed as a human-readable companion tag. The
|
||||
legacy Kubernetes rollback pool is currently disabled and has no labels.
|
||||
If it is restored, its Nix-capable image must be configured separately and
|
||||
digest-pinned before enabling a `nix` label.
|
||||
|
||||
If the Gitea container registry is unavailable, select a private registry that
|
||||
is reachable from the runner Kubernetes cluster and requires authentication that
|
||||
can be provided through Kubernetes image-pull secrets. Record the selected
|
||||
registry and replace the host in the same digest-pinned form:
|
||||
|
||||
```text
|
||||
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
|
||||
```
|
||||
|
||||
Do not fall back to `latest` or a tag-only mapping.
|
||||
Do not use a tag-only image for a restored Kubernetes rollback pool.
|
||||
|
||||
## Task 7 publication status
|
||||
|
||||
Local build evidence is recorded in
|
||||
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
|
||||
load and tag the image, but pushing to the preferred registry failed with
|
||||
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
|
||||
concrete final mapping. Kubernetes pull smoke is recorded in
|
||||
`.sisyphus/evidence/task-7-image-digest.txt`. Kubernetes pull smoke is recorded in
|
||||
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
|
||||
is not installed or not on `PATH`.
|
||||
|
||||
Once registry credentials are available, rerun the push, capture the
|
||||
registry-reported digest, and replace `<registry-digest>` in the mapping above
|
||||
before Task 6/9 consumes the label configuration.
|
||||
After importing the archive as a Hetzner image, record its image ID in the
|
||||
controller host configuration before dispatching Nix jobs.
|
||||
|
||||
## Image contents
|
||||
|
||||
|
||||
@@ -19,10 +19,8 @@ data:
|
||||
insecure: false
|
||||
fetch_timeout: 5s
|
||||
fetch_interval: 2s
|
||||
labels:
|
||||
- ubuntu-latest
|
||||
# The nix label is intentionally disabled until the runner image has a
|
||||
# concrete registry-reported digest; see ../runbook.md before deploy.
|
||||
# Persistent pool is disabled; controller owns all runner labels.
|
||||
labels: []
|
||||
|
||||
cache:
|
||||
enabled: true
|
||||
|
||||
@@ -45,7 +45,7 @@ variable "control_plane_server_type" {
|
||||
}
|
||||
|
||||
variable "worker_server_type" {
|
||||
description = "Default worker server type for the budget trusted DinD runner pool. One cpx22 worker keeps the idle baseline cheap; scale out later if job pressure requires it."
|
||||
description = "Default worker server type for the budget trusted DinD cluster. One cpx22 worker keeps the cluster baseline cheap; scale out later if rollback capacity requires it."
|
||||
type = string
|
||||
default = "cpx22"
|
||||
}
|
||||
|
||||
@@ -2,15 +2,13 @@
|
||||
|
||||
## Scope
|
||||
|
||||
This directory is the repo-owned boundary for the first Gitea Actions runner
|
||||
pool. Task 1 only establishes the scaffold and immutable decision contract;
|
||||
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
|
||||
and a Nix-capable runner image under the existing subdirectories.
|
||||
This directory is the repo-owned boundary for the Gitea Actions runner pool.
|
||||
The controller is the active on-demand path; Kubernetes manifests and the
|
||||
Nix-capable image are retained for manual rollback and maintenance.
|
||||
|
||||
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
|
||||
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
|
||||
workflow job containers are ephemeral, while each runner pod keeps its runner
|
||||
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||
`hectic-lab`. The pool is trusted-only. "Ephemeral" means each controller VM
|
||||
and workflow job is disposable; the Kubernetes StatefulSet is rollback-only.
|
||||
|
||||
## Immutable decisions
|
||||
|
||||
@@ -27,21 +25,21 @@ identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||
`/data`, including `/data/.runner`.
|
||||
- Container builds run through privileged rootful DinD inside trusted runner
|
||||
pods; host Docker socket mounting is not an implementation path.
|
||||
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
|
||||
the Nix-capable image has been pushed and a concrete registry-reported digest
|
||||
is added to the runner ConfigMap.
|
||||
- `ubuntu-latest` and `nix` are controller-managed on-demand aliases for
|
||||
`gross-x86` and `gross-nix-x86`; the Kubernetes pool has no active labels.
|
||||
- First scope is trusted internal workflows only, with no untrusted fork or PR
|
||||
workflow support.
|
||||
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
|
||||
- On-demand allocation is handled by the repo-owned controller; Kubernetes is
|
||||
not an active autoscaling path.
|
||||
|
||||
## Lifecycle boundaries
|
||||
|
||||
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
|
||||
- `infra/gitea-runners/opentofu/`: OpenTofu stack for the S3 backend
|
||||
contract, Hetzner provider configuration, and kube-hetzner module wiring.
|
||||
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
|
||||
- `infra/gitea-runners/k8s/`: rollback-only namespace, ConfigMap, Secret mount,
|
||||
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
|
||||
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
|
||||
handoff; package or flake output changes are outside Task 1.
|
||||
- `infra/gitea-runners/image/`: notes and handoff for the optional Kubernetes
|
||||
rollback image; active on-demand Nix image is selected by Hetzner image ID.
|
||||
- `infra/gitea-runners/runbook.md`: this contract plus later operational
|
||||
commands, rollback notes, and acceptance evidence references.
|
||||
|
||||
@@ -52,8 +50,8 @@ identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||
instructions, or GitHub Actions ARC assumptions.
|
||||
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
|
||||
acceptable for trusted internal jobs.
|
||||
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
|
||||
runner pool.
|
||||
- The persistent StatefulSet is rollback-only and defaults to zero replicas;
|
||||
normal jobs use controller-managed on-demand VMs.
|
||||
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
|
||||
S3 credentials, decrypted SOPS files, or SOPS age keys.
|
||||
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
|
||||
@@ -248,8 +246,7 @@ These commands are executable only when the external inputs are available:
|
||||
- S3 backend credentials and endpoint access
|
||||
- a matching SOPS age identity for `sus/gitea-runners.yaml`
|
||||
- `kubectl` access to the target cluster
|
||||
- a concrete digest for the pushed Nix-capable runner image, if enabling the
|
||||
`nix` label
|
||||
- a valid `GCR_NIX_IMAGE_ID` for controller-managed Nix jobs
|
||||
|
||||
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
|
||||
stale kubeconfig files.
|
||||
@@ -260,15 +257,12 @@ Before production Kubernetes apply or rollout, satisfy both manifest gates:
|
||||
Kustomize overlay intentionally does not include a placeholder Secret, but
|
||||
the StatefulSet still mounts `secretName: gitea-runner-token` as
|
||||
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
|
||||
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
|
||||
image has been pushed successfully. Enable the `nix` label only by adding a
|
||||
digest-pinned `docker://` mapping with the exact registry-reported sha256
|
||||
digest from that push.
|
||||
2. Keep the persistent-pool ConfigMap labels empty. Runner labels belong to the
|
||||
controller; Nix image readiness is governed by `GCR_NIX_IMAGE_ID`.
|
||||
|
||||
Use the same SOPS materialization pattern as token rotation before applying the
|
||||
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
|
||||
target namespace; the full overlay remains gated on the Secret and digest
|
||||
decisions:
|
||||
target namespace; the full overlay remains gated on the Secret.
|
||||
|
||||
```sh
|
||||
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||
@@ -283,8 +277,7 @@ kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||
```
|
||||
|
||||
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
|
||||
above succeeds. Do not claim or enable the `nix` runner label until the image
|
||||
publication step has produced the concrete digest.
|
||||
above succeeds. The persistent pool ConfigMap must retain empty labels.
|
||||
|
||||
```sh
|
||||
tofu -chdir=infra/gitea-runners/opentofu init
|
||||
@@ -309,14 +302,14 @@ Expected status after deploy:
|
||||
- `kubectl config current-context` names the runner cluster context.
|
||||
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
|
||||
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
|
||||
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
|
||||
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
|
||||
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
|
||||
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 0 desired and 0 ready replicas.
|
||||
- `kubectl -n gitea-runners get pvc` shows no active runner PVCs; retained PVCs are rollback-only.
|
||||
- The controller host reports healthy and owns runner registrations; no persistent runner claims `ubuntu-latest` or `nix`.
|
||||
|
||||
## Scale 5 to 10 to 5
|
||||
## Legacy rollback pool scaling (manual only)
|
||||
|
||||
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
|
||||
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
|
||||
Persistent-pool scaling is not part of normal operation. Use only after restoring
|
||||
its labels and disabling the zero-idle controller as described in `Rollback`.
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
|
||||
@@ -324,9 +317,9 @@ kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
|
||||
# Run the concurrent smoke workflows now.
|
||||
# Run only workflows supported by restored persistent labels.
|
||||
|
||||
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=1
|
||||
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
@@ -405,7 +398,7 @@ Do not run the delete command for a runner that still has an active
|
||||
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
|
||||
being intentionally reset for re-registration.
|
||||
|
||||
## Application rollback
|
||||
## Legacy Kubernetes application rollback
|
||||
|
||||
Rollback the app layer only. Do not use this section to destroy the cluster.
|
||||
|
||||
@@ -504,7 +497,7 @@ as complete.
|
||||
|
||||
## Ephemeral VM runner cutover
|
||||
|
||||
This section governs replacing the fixed K8s runner pool with the
|
||||
This section governs replacing the legacy persistent K8s runner pool with the
|
||||
ephemeral-VM controller (`package/gitea-runner-controller`) on this host.
|
||||
The K8s pool above remains rollback-only until cutover is explicitly accepted.
|
||||
|
||||
@@ -569,10 +562,11 @@ The `deploy-neuro` workflow uses these nested limits:
|
||||
| `gross-nix-x86-perf` runner | 480 minutes |
|
||||
| `gross-nix-x86-highmem` runner | 480 minutes |
|
||||
| Gitea `actions.ENDLESS_TASK_TIMEOUT` | 8 hours |
|
||||
| VM hard lifetime from allocation | 480 minutes plus 10-minute controller grace |
|
||||
| VM hard lifetime from allocation | 480 minutes; no destruction grace |
|
||||
|
||||
Other runner labels keep their existing 180-minute limits. Deploy the controller
|
||||
and Gitea watchdog settings before dispatching the longer workflow. Already
|
||||
`ubuntu-latest` keeps a 180-minute limit; `nix` uses a 480-minute limit for
|
||||
long-running Nix deployments. Deploy the controller and Gitea watchdog settings
|
||||
before dispatching the longer workflow. Already
|
||||
allocated VMs retain the TTL and runner configuration assigned when they were
|
||||
created; updating the controller does not extend a running job.
|
||||
|
||||
@@ -582,10 +576,27 @@ only, never to a lower-RAM server type. Current Hetzner public pricing for
|
||||
Germany/Finland CCX53 is 0.8550 EUR/hour excluding IPv4, so one 480-minute
|
||||
allocation reserves 6.84 EUR against the controller budget before VM creation.
|
||||
|
||||
These are maximum lifetimes: terminal jobs still trigger immediate VM teardown.
|
||||
The controller's budget reservation uses the full label TTL, so a long-running
|
||||
label reserves more of the existing monthly budget. Do not raise that budget or
|
||||
disable timeout safeguards just to bypass a refused allocation.
|
||||
These are maximum lifetimes. Failed, cancelled, skipped, and unbootstrapped jobs
|
||||
still trigger immediate VM teardown; failed jobs retain pre-destroy diagnostics.
|
||||
After a successful job, its bootstrapped VM stays running until next hourly
|
||||
boundary measured from original VM creation, capped by profile TTL. Same-repo,
|
||||
same-label queued work can atomically claim that idle VM. Reuse preserves
|
||||
original Hetzner labels and runner name, creates no server, fetches no new
|
||||
registration token, and makes no second budget reservation. Idle VMs are still
|
||||
billed: controller deletes them at slot expiry and never relies on stopping a
|
||||
server to avoid charges.
|
||||
|
||||
Reuse retains runner host filesystem and registration identity. It is therefore
|
||||
restricted to same repository and exact label inside this trusted-only pool;
|
||||
allowed repositories must not run untrusted fork or pull-request code. A failed
|
||||
or otherwise non-successful job is never reused.
|
||||
|
||||
Active and idle VMs are deleted at profile hard TTL without grace. Idle reuse is
|
||||
allowed only when at least one configured reconcile interval remains before both
|
||||
slot expiry and hard TTL. Controller budget reservation still uses full label TTL
|
||||
on initial creation, so a long-running label reserves more of existing monthly
|
||||
budget. Do not raise that budget or disable timeout safeguards just to bypass a
|
||||
refused allocation.
|
||||
|
||||
After changing any timeout, verify the complete chain rather than only
|
||||
`timeout-minutes`; a shorter wrapper, runner, server watchdog, or VM TTL wins.
|
||||
@@ -598,7 +609,9 @@ journalctl -u gitea-runner-webhook -n 20 --no-pager
|
||||
hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed")] | length' # expect 0
|
||||
```
|
||||
|
||||
Zero managed VMs at idle is the steady-state assertion.
|
||||
Zero managed VMs is expected after retained billing slots expire. Immediately
|
||||
after successful work, one managed VM per retained profile may remain until its
|
||||
recorded hourly boundary.
|
||||
|
||||
### End-to-end acceptance (Task 9)
|
||||
|
||||
@@ -608,10 +621,10 @@ Trigger `.gitea/workflows/runner-nix-smoke.yaml` via workflow_dispatch, then:
|
||||
watch_labels() { hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed") | {id,name,labels}]'; }
|
||||
watch_labels # exactly one VM while queued/running
|
||||
journalctl -f -u gitea-runner-controller # vm-created / vm-destroyed events
|
||||
watch_labels # expect [] after completion
|
||||
watch_labels # VM may remain until next hourly boundary
|
||||
curl -fsS -H "Authorization: token $ADMIN" \
|
||||
https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners \
|
||||
| jq '[.entries[] | select(.name | startswith("gcr-"))] | length' # expect 0
|
||||
| jq '[.entries[] | select(.name | startswith("gcr-"))] | length' # may remain during retained slot
|
||||
```
|
||||
|
||||
Failure paths to verify identically: duplicate delivery (send same webhook twice
|
||||
@@ -628,23 +641,27 @@ K8s rollback pool now defaults to deleted state:
|
||||
Re-enable sequence:
|
||||
|
||||
```sh
|
||||
# 1. stop ephemeral path
|
||||
sed -i 's/hectic.services.gitea-runner-controller = {.*}/\/* disabled *\//' \
|
||||
nixos/system/hectic-lab/hectic-lab.nix # or set enable = false
|
||||
# 1. edit nixos/system/hectic-lab/hectic-lab.nix and set
|
||||
# services.gitea-runner-controller.enable = false, then rebuild:
|
||||
nixos-rebuild --target root@128.140.75.58 switch
|
||||
|
||||
# 2. reprovision old kube-hetzner nodes when they were deleted:
|
||||
tofu -chdir=infra/gitea-runners/opentofu apply
|
||||
|
||||
# 3. restore kubeconfig / cluster access, then re-enable K8s runner pool:
|
||||
# 3. while controller is disabled, destroy every surviving managed VM and
|
||||
# verify no gcr-* runner registration remains online:
|
||||
hcloud server list -o json \
|
||||
| jq -r '.[] | select(.labels["gitea-runner-controller"]=="managed") | .id' \
|
||||
| xargs -r -n1 hcloud server delete
|
||||
|
||||
# 4. restore kubeconfig / cluster access, restore `ubuntu-latest` in the
|
||||
# ConfigMap labels, then re-enable K8s runner pool. The legacy image does not
|
||||
# provide `nix`; do not dispatch Nix workflows until a Nix-capable K8s image
|
||||
# and label mapping are restored:
|
||||
kubectl -n gitea-runners edit configmap/gitea-runner-config
|
||||
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
```
|
||||
|
||||
Any surviving ephemeral VMs after step 1 must be destroyed manually once:
|
||||
|
||||
```sh
|
||||
hcloud server list -o json \
|
||||
| jq -r '.[] | select(.labels["gitea-runner-controller"]=="managed") | .id' \
|
||||
| xargs -r -n1 hcloud server delete
|
||||
```
|
||||
Any managed VM or `gcr-*` registration found after step 3 must be removed before
|
||||
restoring persistent labels; otherwise both pools can claim the same job.
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ in {
|
||||
else throw (envErrorMessage varName);
|
||||
|
||||
# -- Cargo.toml --
|
||||
cargoToml = src: (builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"));
|
||||
cargoToml = manifest: (builtins.fromTOML (builtins.readFile manifest));
|
||||
|
||||
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
|
||||
# for everything that creates objects in the `hectic` namespace, used by
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
{ ... }:
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
cfg = config.services.minecraft-public-relay;
|
||||
in {
|
||||
options.services.minecraft-public-relay = {
|
||||
enable = lib.mkEnableOption "restricted SSH relay for Minecraft";
|
||||
publicPort = lib.mkOption { type = lib.types.port; default = 25568; };
|
||||
tunnelPort = lib.mkOption { type = lib.types.port; default = 25577; };
|
||||
publicKey = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public SSH key of the Minecraft tunnel client";
|
||||
};
|
||||
};
|
||||
config = lib.mkIf cfg.enable {
|
||||
networking.firewall.allowedTCPPorts = [ cfg.publicPort ];
|
||||
users.groups.mc-wow-relay = { };
|
||||
users.users.mc-wow-relay = {
|
||||
isSystemUser = true;
|
||||
group = "mc-wow-relay";
|
||||
openssh.authorizedKeys.keys = [
|
||||
"restrict,port-forwarding,permitlisten=\"127.0.0.1:${toString cfg.tunnelPort}\" ${cfg.publicKey}"
|
||||
];
|
||||
};
|
||||
services.openssh.extraConfig = ''
|
||||
Match User mc-wow-relay
|
||||
ClientAliveInterval 15
|
||||
ClientAliveCountMax 3
|
||||
AllowTcpForwarding remote
|
||||
PermitListen 127.0.0.1:${toString cfg.tunnelPort}
|
||||
AllowAgentForwarding no
|
||||
X11Forwarding no
|
||||
PermitTTY no
|
||||
ForceCommand ${pkgs.coreutils}/bin/false
|
||||
Match all
|
||||
'';
|
||||
systemd.sockets.minecraft-wow-proxy = {
|
||||
description = "WorldOfSosal WoW public Minecraft port";
|
||||
wantedBy = [ "sockets.target" ];
|
||||
listenStreams = [ "0.0.0.0:${toString cfg.publicPort}" ];
|
||||
};
|
||||
systemd.services.minecraft-wow-proxy = {
|
||||
description = "Forward Minecraft to the neuro reverse tunnel";
|
||||
requires = [ "minecraft-wow-proxy.socket" ];
|
||||
after = [ "minecraft-wow-proxy.socket" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd 127.0.0.1:${toString cfg.tunnelPort}";
|
||||
DynamicUser = true;
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -21,6 +21,13 @@
|
||||
plugin_daemon:
|
||||
ports: !override
|
||||
- "127.0.0.1:${toString cfg.pluginPort}:5003"
|
||||
environment:
|
||||
DB_USERNAME: ''${DB_USERNAME:-postgres}
|
||||
DB_HOST: ''${DB_HOST:-db_postgres}
|
||||
DB_PORT: ''${DB_PORT:-5432}
|
||||
REDIS_HOST: ''${REDIS_HOST:-redis}
|
||||
REDIS_PORT: ''${REDIS_PORT:-6379}
|
||||
REDIS_DB: ''${REDIS_DB:-0}
|
||||
'';
|
||||
in {
|
||||
options.hectic.services.dify = {
|
||||
|
||||
@@ -92,12 +92,12 @@ in
|
||||
concurrencyCap = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 2;
|
||||
description = "Maximum simultaneously running ephemeral VMs (global).";
|
||||
description = "Maximum simultaneously assigned runner VMs (global); retained idle VMs do not count.";
|
||||
};
|
||||
perRepoCap = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 1;
|
||||
description = "Maximum concurrent ephemeral VMs per repo.";
|
||||
description = "Maximum concurrently assigned runner VMs per repo; retained idle VMs do not count.";
|
||||
};
|
||||
reconcileIntervalSec = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
|
||||
@@ -59,25 +59,25 @@ in
|
||||
|
||||
host = lib.mkOption {
|
||||
type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9.-]*";
|
||||
default = "u666713.your-storagebox.de";
|
||||
default = "u666713-sub1.your-storagebox.de";
|
||||
description = "Hetzner Storage Box SMB hostname.";
|
||||
};
|
||||
|
||||
username = lib.mkOption {
|
||||
type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_-]*";
|
||||
default = "u666713";
|
||||
default = "u666713-sub1";
|
||||
description = "Storage Box SMB username.";
|
||||
};
|
||||
|
||||
share = lib.mkOption {
|
||||
type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_-]*";
|
||||
default = "backup";
|
||||
default = "u666713-sub1";
|
||||
description = "SMB share exported by Storage Box.";
|
||||
};
|
||||
|
||||
subdirectory = lib.mkOption {
|
||||
type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_./-]*";
|
||||
default = "immich";
|
||||
type = lib.types.nullOr (lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_./-]*");
|
||||
default = null;
|
||||
description = "Directory within the SMB share used by Immich.";
|
||||
};
|
||||
|
||||
@@ -146,8 +146,7 @@ in
|
||||
options = [
|
||||
"_netdev"
|
||||
"nofail"
|
||||
"x-systemd.automount"
|
||||
"x-systemd.idle-timeout=600"
|
||||
"x-systemd.mount-timeout=60s"
|
||||
"vers=3.1.1"
|
||||
"seal"
|
||||
"cache=none"
|
||||
@@ -157,12 +156,13 @@ in
|
||||
"gid=${config.services.immich.group}"
|
||||
"file_mode=0660"
|
||||
"dir_mode=0770"
|
||||
"prefixpath=${cfg.storageBox.subdirectory}"
|
||||
];
|
||||
] ++ lib.optional (cfg.storageBox.subdirectory != null)
|
||||
"prefixpath=${cfg.storageBox.subdirectory}";
|
||||
};
|
||||
|
||||
systemd.services.immich-server.serviceConfig.RequiresMountsFor = lib.mkIf cfg.storageBox.enable [
|
||||
systemd.services.immich-server.unitConfig.RequiresMountsFor = lib.mkIf cfg.storageBox.enable [
|
||||
cfg.mediaLocation
|
||||
];
|
||||
systemd.services.immich-server.serviceConfig.Restart = lib.mkForce "always";
|
||||
};
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
"-L" "${cfg.dataDir}/logs/bootstrap.log"
|
||||
] ++ lib.optional (!cfg.caseSensitive) "-C1";
|
||||
|
||||
initScript = pkgs.writeShellScript "p4d-init" ''
|
||||
initScript = ''
|
||||
set -eu
|
||||
|
||||
export P4ROOT=${lib.escapeShellArg cfg.dataDir}
|
||||
@@ -334,7 +334,7 @@ in {
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
preStart = builtins.readFile initScript;
|
||||
preStart = initScript;
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = serviceUser;
|
||||
|
||||
@@ -16,7 +16,183 @@
|
||||
name: value:
|
||||
"${name}=${if builtins.isBool value then lib.boolToString value else toString value}"
|
||||
) serverProperties;
|
||||
sandboxValueType = lib.types.oneOf [
|
||||
lib.types.bool
|
||||
lib.types.int
|
||||
lib.types.float
|
||||
lib.types.str
|
||||
(lib.types.attrsOf sandboxValueType)
|
||||
];
|
||||
luaValue = value:
|
||||
if builtins.isBool value then
|
||||
lib.boolToString value
|
||||
else if builtins.isInt value || builtins.isFloat value then
|
||||
toString value
|
||||
else if builtins.isAttrs value then
|
||||
"{ ${lib.concatStringsSep " " (lib.mapAttrsToList (name: child: "[${luaValue name}] = ${luaValue child},") value)} }"
|
||||
else
|
||||
"\"${lib.replaceStrings [ "\\" "\"" "\n" "\r" ] [ "\\\\" "\\\"" "\\n" "\\r" ] value}\"";
|
||||
sandboxConfigLines = lib.mapAttrsToList (
|
||||
name: value: "[${luaValue name}] = ${luaValue value},"
|
||||
) cfg.sandboxProperties;
|
||||
zomboidDir = "${cfg.dataDir}/Zomboid";
|
||||
adminPasswordFile = "${cfg.dataDir}/admin-password";
|
||||
rconPasswordFile = cfg.rcon.passwordFile;
|
||||
backupCfg = cfg.backup;
|
||||
s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile;
|
||||
s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
|
||||
s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint;
|
||||
s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region;
|
||||
saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}";
|
||||
serverConfigDir = "${zomboidDir}/Server";
|
||||
backupScript = pkgs.writeShellScript "project-zomboid-backup" ''
|
||||
set -eu
|
||||
|
||||
staging_dir=${lib.escapeShellArg backupCfg.stagingDir}
|
||||
archive_dir=${lib.escapeShellArg backupCfg.archiveDir}
|
||||
lock_file="$archive_dir/.backup.lock"
|
||||
|
||||
${pkgs.coreutils}/bin/install -d -m 0700 \
|
||||
"$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}" \
|
||||
"$staging_dir/Zomboid/Server" \
|
||||
"$archive_dir"
|
||||
|
||||
exec 9>"$lock_file"
|
||||
if ! ${pkgs.util-linux}/bin/flock -n 9; then
|
||||
${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid backup already running; skipping.' >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
${lib.optionalString cfg.rcon.enable ''
|
||||
rcon_password="$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})"
|
||||
if [ -z "$rcon_password" ]; then
|
||||
${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid RCON password file is empty.' >&2
|
||||
exit 1
|
||||
fi
|
||||
${pkgs.rcon}/bin/rcon \
|
||||
--host 127.0.0.1 \
|
||||
--port ${toString cfg.rcon.port} \
|
||||
--password "$rcon_password" \
|
||||
save
|
||||
${pkgs.coreutils}/bin/sleep ${toString backupCfg.saveWaitSeconds}
|
||||
''}
|
||||
|
||||
sync_staging() {
|
||||
${pkgs.rsync}/bin/rsync -a --delete \
|
||||
${lib.escapeShellArg "${saveDir}/"} \
|
||||
"$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}/"
|
||||
${pkgs.rsync}/bin/rsync -a --delete --delete-excluded \
|
||||
--include=${lib.escapeShellArg "/${cfg.serverName}_SandboxVars.lua"} \
|
||||
--include=${lib.escapeShellArg "/${cfg.serverName}_spawnpoints.lua"} \
|
||||
--include=${lib.escapeShellArg "/${cfg.serverName}_spawnregions.lua"} \
|
||||
--exclude='*' \
|
||||
${lib.escapeShellArg "${serverConfigDir}/"} \
|
||||
"$staging_dir/Zomboid/Server/"
|
||||
}
|
||||
|
||||
# Second pass narrows, but cannot eliminate, live-save inconsistency.
|
||||
sync_staging
|
||||
${pkgs.coreutils}/bin/sleep 5
|
||||
sync_staging
|
||||
|
||||
timestamp="$(${pkgs.coreutils}/bin/date -u +%Y%m%dT%H%M%SZ)"
|
||||
archive_name="project-zomboid-${cfg.serverName}-$timestamp.tar.zst"
|
||||
archive_tmp="$archive_dir/.$archive_name.tmp"
|
||||
archive="$archive_dir/$archive_name"
|
||||
trap '${pkgs.coreutils}/bin/rm -f "$archive_tmp"' EXIT
|
||||
${pkgs.gnutar}/bin/tar \
|
||||
--use-compress-program=${lib.escapeShellArg "${pkgs.zstd}/bin/zstd -T0"} \
|
||||
-C "$staging_dir" -cf "$archive_tmp" Zomboid
|
||||
${pkgs.coreutils}/bin/chmod 0600 "$archive_tmp"
|
||||
${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive"
|
||||
trap - EXIT
|
||||
|
||||
${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
|
||||
-name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \
|
||||
-mmin +${toString (backupCfg.retentionDays * 1440)} -delete
|
||||
|
||||
${lib.optionalString backupCfg.s3.enable ''
|
||||
if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
|
||||
${pkgs.coreutils}/bin/printf '%s\n' \
|
||||
'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
|
||||
exit 1
|
||||
fi
|
||||
s3_bucket=${lib.escapeShellArg s3Bucket}
|
||||
s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
|
||||
s3_key="''${s3_prefix:+$s3_prefix/}$archive_name"
|
||||
${pkgs.awscli2}/bin/aws s3 cp "$archive" \
|
||||
"s3://$s3_bucket/$s3_key" \
|
||||
--endpoint-url ${lib.escapeShellArg s3Endpoint} \
|
||||
--region ${lib.escapeShellArg s3Region} \
|
||||
--cli-connect-timeout 30 \
|
||||
--cli-read-timeout 300 \
|
||||
--only-show-errors
|
||||
|
||||
remote_prefix="$s3_prefix"
|
||||
if [ -n "$remote_prefix" ]; then
|
||||
remote_prefix="$remote_prefix/"
|
||||
fi
|
||||
archive_prefix=${lib.escapeShellArg "project-zomboid-${cfg.serverName}-"}
|
||||
remote_list="$staging_dir/.remote-objects.json"
|
||||
remote_delete_dir="$staging_dir/.remote-delete"
|
||||
${pkgs.awscli2}/bin/aws s3api list-objects-v2 \
|
||||
--bucket "$s3_bucket" \
|
||||
--prefix "$remote_prefix" \
|
||||
--endpoint-url ${lib.escapeShellArg s3Endpoint} \
|
||||
--region ${lib.escapeShellArg s3Region} \
|
||||
--output json > "$remote_list"
|
||||
${pkgs.python3}/bin/python3 - "$remote_list" "$remote_delete_dir" \
|
||||
"$(( $(${pkgs.coreutils}/bin/date +%s) - ${toString (backupCfg.s3.remoteRetentionDays * 86400)} ))" \
|
||||
"$remote_prefix$archive_prefix" <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
objects_path, delete_dir, cutoff, key_prefix = sys.argv[1:]
|
||||
cutoff = int(cutoff)
|
||||
archive_pattern = re.compile(
|
||||
re.escape(key_prefix) + r"\d{8}T\d{6}Z\.tar\.zst\Z"
|
||||
)
|
||||
with open(objects_path, encoding="utf-8") as stream:
|
||||
objects = json.load(stream).get("Contents", [])
|
||||
|
||||
old_keys = []
|
||||
for item in objects:
|
||||
key = item.get("Key", "")
|
||||
if not archive_pattern.fullmatch(key):
|
||||
continue
|
||||
modified = datetime.datetime.fromisoformat(
|
||||
item["LastModified"].replace("Z", "+00:00")
|
||||
)
|
||||
if int(modified.timestamp()) < cutoff:
|
||||
old_keys.append(key)
|
||||
|
||||
os.makedirs(delete_dir, exist_ok=True)
|
||||
for batch_number in range(0, len(old_keys), 1000):
|
||||
batch = old_keys[batch_number:batch_number + 1000]
|
||||
manifest_path = os.path.join(
|
||||
delete_dir, f"batch-{batch_number // 1000:04d}.json"
|
||||
)
|
||||
with open(manifest_path, "w", encoding="utf-8") as stream:
|
||||
json.dump(
|
||||
{"Objects": [{"Key": key} for key in batch], "Quiet": True},
|
||||
stream,
|
||||
)
|
||||
PY
|
||||
for remote_manifest in "$remote_delete_dir"/*.json; do
|
||||
[ -f "$remote_manifest" ] || continue
|
||||
${pkgs.awscli2}/bin/aws s3api delete-objects \
|
||||
--bucket "$s3_bucket" \
|
||||
--delete "file://$remote_manifest" \
|
||||
--endpoint-url ${lib.escapeShellArg s3Endpoint} \
|
||||
--region ${lib.escapeShellArg s3Region} \
|
||||
--only-show-errors
|
||||
done
|
||||
${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir"
|
||||
''}
|
||||
'';
|
||||
startScript = pkgs.writeShellScript "project-zomboid-start" ''
|
||||
admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile})
|
||||
exec ${pkgs.steam-run}/bin/steam-run \
|
||||
@@ -100,14 +276,160 @@ in {
|
||||
description = "Runtime file with additional INI values, suitable for secrets.";
|
||||
};
|
||||
|
||||
sandboxProperties = lib.mkOption {
|
||||
type = lib.types.attrsOf sandboxValueType;
|
||||
default = { };
|
||||
description = "Values for the Project Zomboid SandboxVars.lua file.";
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Open the Project Zomboid UDP ports in the firewall.";
|
||||
};
|
||||
|
||||
rcon = {
|
||||
enable = lib.mkEnableOption "local RCON for Project Zomboid automation";
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 27015;
|
||||
description = "RCON TCP port; not opened in the firewall by this module.";
|
||||
};
|
||||
|
||||
passwordFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = "${cfg.dataDir}/rcon-password";
|
||||
description = "Runtime file containing the generated RCON password.";
|
||||
};
|
||||
};
|
||||
|
||||
backup = {
|
||||
enable = lib.mkEnableOption "no-stop Project Zomboid backups";
|
||||
|
||||
onCalendar = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "*:0/30";
|
||||
description = "systemd calendar expression controlling backup frequency.";
|
||||
};
|
||||
|
||||
stagingDir = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = "${cfg.dataDir}/backups/staging";
|
||||
description = "Local directory containing the two-pass rsync staging tree.";
|
||||
};
|
||||
|
||||
archiveDir = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = "${cfg.dataDir}/backups/archive";
|
||||
description = "Local directory containing timestamped tar.zst archives.";
|
||||
};
|
||||
|
||||
retentionDays = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 14;
|
||||
description = "Delete local archives older than this many days.";
|
||||
};
|
||||
|
||||
saveWaitSeconds = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 10;
|
||||
description = "Seconds to wait after the RCON save command before rsync.";
|
||||
};
|
||||
|
||||
s3 = {
|
||||
enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage";
|
||||
|
||||
credentialsFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = ''
|
||||
Runtime env file containing AWS_ACCESS_KEY_ID and
|
||||
AWS_SECRET_ACCESS_KEY. Required when S3 upload is enabled.
|
||||
'';
|
||||
};
|
||||
|
||||
bucket = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "S3 bucket receiving backup archives.";
|
||||
};
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "S3-compatible endpoint URL.";
|
||||
};
|
||||
|
||||
region = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "S3 region passed to awscli2.";
|
||||
};
|
||||
|
||||
prefix = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "project-zomboid";
|
||||
description = "Optional object key prefix within the S3 bucket.";
|
||||
};
|
||||
|
||||
remoteRetentionDays = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 14;
|
||||
description = "Delete uploaded archives older than this many days.";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = !cfg.rcon.enable || (
|
||||
lib.hasPrefix "/" cfg.rcon.passwordFile
|
||||
&& !lib.hasPrefix "/nix/store/" cfg.rcon.passwordFile
|
||||
);
|
||||
message = "hectic.services.project-zomboid.rcon.passwordFile must be a runtime path outside /nix/store.";
|
||||
}
|
||||
{
|
||||
assertion = !backupCfg.s3.enable || backupCfg.enable;
|
||||
message = "hectic.services.project-zomboid.backup must be enabled before S3 upload.";
|
||||
}
|
||||
{
|
||||
assertion = !backupCfg.s3.enable || backupCfg.s3.credentialsFile != null;
|
||||
message = "hectic.services.project-zomboid.backup.s3.credentialsFile is required when S3 upload is enabled.";
|
||||
}
|
||||
{
|
||||
assertion = !backupCfg.s3.enable || backupCfg.s3.bucket != null;
|
||||
message = "hectic.services.project-zomboid.backup.s3.bucket is required when S3 upload is enabled.";
|
||||
}
|
||||
{
|
||||
assertion = !backupCfg.s3.enable || backupCfg.s3.endpoint != null;
|
||||
message = "hectic.services.project-zomboid.backup.s3.endpoint is required when S3 upload is enabled.";
|
||||
}
|
||||
{
|
||||
assertion = !backupCfg.s3.enable || backupCfg.s3.region != null;
|
||||
message = "hectic.services.project-zomboid.backup.s3.region is required when S3 upload is enabled.";
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
!backupCfg.s3.enable
|
||||
|| backupCfg.s3.credentialsFile == null
|
||||
|| (
|
||||
lib.hasPrefix "/" backupCfg.s3.credentialsFile
|
||||
&& !lib.hasPrefix "/nix/store/" backupCfg.s3.credentialsFile
|
||||
);
|
||||
message = "hectic.services.project-zomboid.backup.s3.credentialsFile must be a runtime path outside /nix/store.";
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
!backupCfg.s3.enable
|
||||
|| backupCfg.s3.endpoint == null
|
||||
|| lib.hasPrefix "https://" backupCfg.s3.endpoint;
|
||||
message = "hectic.services.project-zomboid.backup.s3.endpoint must use HTTPS.";
|
||||
}
|
||||
];
|
||||
|
||||
users.groups.project-zomboid = { };
|
||||
users.users.project-zomboid = {
|
||||
isSystemUser = true;
|
||||
@@ -119,6 +441,11 @@ in {
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.dataDir} 0750 project-zomboid project-zomboid - -"
|
||||
"d ${cfg.installDir} 0750 project-zomboid project-zomboid - -"
|
||||
] ++ lib.optionals backupCfg.enable [
|
||||
"d ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
|
||||
"Z ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
|
||||
"d ${backupCfg.stagingDir} 0700 project-zomboid project-zomboid - -"
|
||||
"d ${backupCfg.archiveDir} 0700 project-zomboid project-zomboid - -"
|
||||
];
|
||||
|
||||
systemd.services.project-zomboid = {
|
||||
@@ -134,6 +461,22 @@ in {
|
||||
umask 077
|
||||
${pkgs.openssl}/bin/openssl rand -base64 32 > ${lib.escapeShellArg adminPasswordFile}
|
||||
fi
|
||||
${lib.optionalString cfg.rcon.enable ''
|
||||
if [ ! -s ${lib.escapeShellArg rconPasswordFile} ]; then
|
||||
umask 077
|
||||
${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile}
|
||||
else
|
||||
rcon_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})
|
||||
case "$rcon_password" in
|
||||
*[!0123456789abcdefABCDEF]*)
|
||||
umask 077
|
||||
${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile}
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
${pkgs.coreutils}/bin/chown project-zomboid:project-zomboid ${lib.escapeShellArg rconPasswordFile}
|
||||
${pkgs.coreutils}/bin/chmod 0600 ${lib.escapeShellArg rconPasswordFile}
|
||||
''}
|
||||
${pkgs.steamcmd}/bin/steamcmd \
|
||||
+force_install_dir ${lib.escapeShellArg cfg.installDir} \
|
||||
+login anonymous \
|
||||
@@ -143,14 +486,33 @@ in {
|
||||
's/"-Xmx[0-9]+[mMgG]"/"-Xmx${cfg.memory}"/' \
|
||||
${lib.escapeShellArg "${cfg.installDir}/ProjectZomboid64.json"}
|
||||
${pkgs.coreutils}/bin/install -d -m 0750 \
|
||||
${lib.escapeShellArg "${cfg.dataDir}/Server"}
|
||||
${lib.escapeShellArg "${zomboidDir}/Server"}
|
||||
{
|
||||
${lib.concatMapStringsSep "\n " (line:
|
||||
"${pkgs.coreutils}/bin/printf '%s\\n' ${lib.escapeShellArg line};"
|
||||
) configLines}
|
||||
${lib.optionalString (cfg.serverPropertiesFile != null)
|
||||
"${pkgs.coreutils}/bin/cat ${lib.escapeShellArg cfg.serverPropertiesFile};"}
|
||||
} > ${lib.escapeShellArg "${cfg.dataDir}/Server/${cfg.serverName}.ini"}
|
||||
${lib.optionalString cfg.rcon.enable ''
|
||||
${pkgs.coreutils}/bin/printf '%s\n' ${lib.escapeShellArg "RCONPort=${toString cfg.rcon.port}"};
|
||||
${pkgs.coreutils}/bin/printf '%s' 'RCONPassword=';
|
||||
${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile};
|
||||
${pkgs.coreutils}/bin/printf '\n';
|
||||
''}
|
||||
} > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}.ini"}
|
||||
${lib.optionalString (cfg.sandboxProperties != { }) ''
|
||||
{
|
||||
${pkgs.coreutils}/bin/printf '%s\n' 'SandboxVars = {';
|
||||
${lib.concatMapStringsSep "\n " (line:
|
||||
"${pkgs.coreutils}/bin/printf '%s\\n' ${lib.escapeShellArg line};"
|
||||
) sandboxConfigLines}
|
||||
${pkgs.coreutils}/bin/printf '%s\n' '};';
|
||||
} > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
|
||||
''}
|
||||
${lib.optionalString (cfg.sandboxProperties == { }) ''
|
||||
${pkgs.coreutils}/bin/rm -f \
|
||||
${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
|
||||
''}
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
@@ -159,16 +521,45 @@ in {
|
||||
WorkingDirectory = cfg.dataDir;
|
||||
Environment = [
|
||||
"HOME=${cfg.dataDir}"
|
||||
"SteamAppId=380870"
|
||||
"SteamAppId=108600"
|
||||
];
|
||||
ExecStart = startScript;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
TimeoutStartSec = "15min";
|
||||
TimeoutStopSec = 30;
|
||||
UMask = "0077";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.project-zomboid-backup = lib.mkIf backupCfg.enable {
|
||||
description = "No-stop Project Zomboid backup";
|
||||
after = [ "project-zomboid.service" ];
|
||||
unitConfig.ConditionPathExists = [
|
||||
saveDir
|
||||
serverConfigDir
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = "project-zomboid";
|
||||
Group = "project-zomboid";
|
||||
ExecStart = backupScript;
|
||||
TimeoutStartSec = "30min";
|
||||
UMask = "0077";
|
||||
} // lib.optionalAttrs backupCfg.s3.enable {
|
||||
EnvironmentFile = s3CredentialsFile;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.project-zomboid-backup = lib.mkIf backupCfg.enable {
|
||||
description = "Run Project Zomboid backup";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = backupCfg.onCalendar;
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [
|
||||
cfg.port
|
||||
cfg.udpPort
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||
in {
|
||||
imports = [
|
||||
./minecraft-wow.nix
|
||||
self.nixosModules.xray-system
|
||||
self.nixosModules.matrix-cluster
|
||||
self.nixosModules.matrix-cluster-users
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{ ... }:
|
||||
{
|
||||
imports = [ (import ../../module/generic/minecraft-public-relay.nix { }) ];
|
||||
services.minecraft-public-relay = {
|
||||
enable = true;
|
||||
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNWWegOVTOF3EOmam32iP7sMybULMTxsXuC+cEGITQ8 minecraft-wow-relay";
|
||||
};
|
||||
systemd.tmpfiles.rules = [ "d /var/www/store/minecraft/world-of-sosal 0755 root root -" ];
|
||||
services.nginx.virtualHosts."store.bfs.band" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
root = "/var/www/store";
|
||||
locations."= /".return = "302 /minecraft/world-of-sosal/";
|
||||
locations."= /minecraft".return = "302 /minecraft/world-of-sosal/";
|
||||
locations."= /minecraft/".return = "302 /minecraft/world-of-sosal/";
|
||||
locations."/".extraConfig = ''
|
||||
autoindex off;
|
||||
add_header Cache-Control "no-cache";
|
||||
try_files $uri $uri/ =404;
|
||||
'';
|
||||
};
|
||||
# Keep old pack URLs working for already imported Prism instances.
|
||||
services.nginx.virtualHosts."bfs.band".locations = {
|
||||
"= /minecraft".return = "302 /minecraft/world-of-sosal/";
|
||||
"= /minecraft/".return = "302 /minecraft/world-of-sosal/";
|
||||
"^~ /minecraft/" = {
|
||||
root = "/var/www/store";
|
||||
extraConfig = ''
|
||||
autoindex off;
|
||||
add_header Cache-Control "no-cache";
|
||||
try_files $uri $uri/ =404;
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -48,6 +48,7 @@ let
|
||||
giteaRunnerService = "gitea-runner-${giteaRunnerEscapedInstance}";
|
||||
giteaRunnerTokenEnvService = "${giteaRunnerService}-token-env";
|
||||
giteaRunnerTokenEnv = "/run/gitea-runner-${giteaRunnerInstance}/token.env";
|
||||
worldOfSosalRoot = "/var/www/store/world-of-sosal";
|
||||
in {
|
||||
imports = [
|
||||
self.nixosModules.hectic
|
||||
@@ -58,11 +59,14 @@ in {
|
||||
self.nixosModules."shadowsocks" # NOTE(nrv): usage/instance
|
||||
|
||||
inputs.hectic-landing.nixosModules.hectic-landing
|
||||
inputs.iana-angl.nixosModules.iana-angl
|
||||
|
||||
(import ./attic.nix { inherit flake self inputs domain; })
|
||||
(import ./containers.nix { inherit flake self inputs; })
|
||||
./experimental-sshd.nix
|
||||
./minecraft-wow-proxy.nix
|
||||
(import ./ente.nix { inherit domain; })
|
||||
(import ./immich.nix { inherit domain; })
|
||||
(import ./mechabellum.nix { inherit flake self inputs domain; })
|
||||
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
|
||||
];
|
||||
@@ -75,6 +79,12 @@ in {
|
||||
host = "127.0.0.1";
|
||||
};
|
||||
|
||||
services.iana-angl = {
|
||||
enable = true;
|
||||
package = inputs.iana-angl.packages.${pkgs.stdenv.hostPlatform.system}.iana-angl;
|
||||
domain = "lessons.${domain}";
|
||||
};
|
||||
|
||||
# NOTE(yukkop): both nixos-mailserver and hectic-landing module set
|
||||
# security.acme.defaults.email. Force the mailserver-aligned address.
|
||||
security.acme.defaults.email = lib.mkForce "security@${domain}";
|
||||
@@ -93,11 +103,81 @@ in {
|
||||
};
|
||||
services."project-zomboid" = {
|
||||
enable = true;
|
||||
memory = "3g";
|
||||
memory = "4g";
|
||||
serverName = "servertest";
|
||||
workshopItems = [ ];
|
||||
mods = [ ];
|
||||
serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
|
||||
rcon.enable = true;
|
||||
backup = {
|
||||
enable = true;
|
||||
onCalendar = "*:0/30";
|
||||
retentionDays = 14;
|
||||
s3 = {
|
||||
enable = true;
|
||||
bucket = "backup-hectic-lab";
|
||||
endpoint = "https://hel1.your-objectstorage.com";
|
||||
region = "hel1";
|
||||
credentialsFile = "/var/lib/project-zomboid/s3-credentials";
|
||||
};
|
||||
};
|
||||
serverProperties = {
|
||||
Map = "Muldraugh, KY";
|
||||
SaveWorldEveryMinutes = 15;
|
||||
DoLuaChecksum = false;
|
||||
Public = true;
|
||||
AntiCheatSafety = 4;
|
||||
AntiCheatMovement = 4;
|
||||
AntiCheatSpeed = 4;
|
||||
AntiCheatHit = 4;
|
||||
AntiCheatPacket = 4;
|
||||
AntiCheatPacketException = 4;
|
||||
AntiCheatPermission = 4;
|
||||
AntiCheatXP = 4;
|
||||
AntiCheatFire = 4;
|
||||
AntiCheatSafeHouse = 4;
|
||||
AntiCheatRecipe = 4;
|
||||
AntiCheatPlayer = 4;
|
||||
AntiCheatChecksum = 4;
|
||||
AntiCheatItem = 4;
|
||||
AntiCheatNoClip = 4;
|
||||
AntiCheatServerCustomization = 4;
|
||||
};
|
||||
workshopItems = [
|
||||
"3676456221" # Lua Digital Watch Framework
|
||||
"3600401184" # Realistic Temperature Mod
|
||||
];
|
||||
mods = [
|
||||
"\\LuaDigitalWatchUI"
|
||||
"\\RC_RealisticColdMod"
|
||||
];
|
||||
sandboxProperties = {
|
||||
StartMonth = 12;
|
||||
StartDay = 1;
|
||||
WaterShut = 3;
|
||||
WaterShutModifier = 150;
|
||||
ElecShut = 3;
|
||||
ElecShutModifier = 150;
|
||||
MinutesPerPage = 0.5;
|
||||
Zombies = 4;
|
||||
ZombieConfig = {
|
||||
PopulationMultiplier = 1.3;
|
||||
PopulationStartMultiplier = 1.0;
|
||||
PopulationPeakMultiplier = 1.0;
|
||||
RespawnHours = 0.0;
|
||||
RespawnUnseenHours = 0.0;
|
||||
RespawnMultiplier = 0.0;
|
||||
RedistributeHours = 0.0;
|
||||
};
|
||||
ZombieLore = {
|
||||
Transmission = 4;
|
||||
Mortality = 7;
|
||||
Speed = 2;
|
||||
SprinterPercentage = 0;
|
||||
Strength = 2;
|
||||
Cognition = 2;
|
||||
DoorOpeningPercentage = 10;
|
||||
};
|
||||
};
|
||||
};
|
||||
services.p4d = {
|
||||
enable = true;
|
||||
package = pkgs.p4d;
|
||||
@@ -109,6 +189,7 @@ in {
|
||||
# NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job).
|
||||
# Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover".
|
||||
enable = true;
|
||||
budgetEurMonthly = "30";
|
||||
imageId = "429747473"; # MicroOS x86 + persistent controller SSH key and writable Nix mount
|
||||
armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot
|
||||
nixImageId = "161547269"; # Ubuntu 24.04 x86; Nix needs writable root
|
||||
@@ -174,7 +255,8 @@ in {
|
||||
key = "init-postgresql";
|
||||
};
|
||||
"atticd/environment" = {};
|
||||
"wg-bfs/private-key" = {};
|
||||
"immich/storage-box" = {};
|
||||
"wg-bfs/private-key" = {};
|
||||
"gitea-runner/org-registration-token" = {
|
||||
sopsFile = flake + "/sus/gitea-runners.yaml";
|
||||
key = "gitea/hectic-lab/org-runner-registration-token";
|
||||
@@ -185,9 +267,50 @@ in {
|
||||
"jwt-secret"
|
||||
"s3-access-key"
|
||||
"s3-secret-key"
|
||||
]);
|
||||
]) // {
|
||||
"project-zomboid/s3-access-key" = {
|
||||
key = "ente/s3-access-key";
|
||||
owner = "project-zomboid";
|
||||
group = "project-zomboid";
|
||||
};
|
||||
"project-zomboid/s3-secret-key" = {
|
||||
key = "ente/s3-secret-key";
|
||||
owner = "project-zomboid";
|
||||
group = "project-zomboid";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.project-zomboid.preStart = lib.mkBefore ''
|
||||
password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"}
|
||||
properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"}
|
||||
s3_credentials_file=${lib.escapeShellArg "/var/lib/project-zomboid/s3-credentials"}
|
||||
s3_credentials_tmp="$(${pkgs.coreutils}/bin/mktemp "''${s3_credentials_file}.XXXXXX")"
|
||||
trap '${pkgs.coreutils}/bin/rm -f "$s3_credentials_tmp"' EXIT
|
||||
|
||||
{
|
||||
${pkgs.coreutils}/bin/printf 'AWS_ACCESS_KEY_ID='
|
||||
${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-access-key".path}
|
||||
${pkgs.coreutils}/bin/printf '\n'
|
||||
${pkgs.coreutils}/bin/printf 'AWS_SECRET_ACCESS_KEY='
|
||||
${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-secret-key".path}
|
||||
${pkgs.coreutils}/bin/printf '\n'
|
||||
} > "$s3_credentials_tmp"
|
||||
${pkgs.coreutils}/bin/chmod 0400 "$s3_credentials_tmp"
|
||||
${pkgs.coreutils}/bin/mv -f "$s3_credentials_tmp" "$s3_credentials_file"
|
||||
|
||||
if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then
|
||||
umask 077
|
||||
${pkgs.openssl}/bin/openssl rand -hex 24 > "$password_file"
|
||||
fi
|
||||
${pkgs.coreutils}/bin/chmod 0600 "$password_file"
|
||||
|
||||
properties_file_tmp="$(${pkgs.coreutils}/bin/mktemp "$(dirname "$properties_file")/.server-password.ini.XXXXXX")"
|
||||
${pkgs.coreutils}/bin/printf 'Password=%s\n' "$(<"$password_file")" > "$properties_file_tmp"
|
||||
${pkgs.coreutils}/bin/chmod 0600 "$properties_file_tmp"
|
||||
${pkgs.coreutils}/bin/mv "$properties_file_tmp" "$properties_file"
|
||||
'';
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
# neuro machine
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
|
||||
@@ -262,6 +385,8 @@ in {
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/www/store 0755 nginx nginx -"
|
||||
"d ${worldOfSosalRoot} 0750 root nginx -"
|
||||
"d ${worldOfSosalRoot}/releases 0750 root nginx -"
|
||||
];
|
||||
|
||||
systemd.services.${giteaRunnerTokenEnvService} = {
|
||||
@@ -306,6 +431,69 @@ in {
|
||||
autoindex on;
|
||||
'';
|
||||
};
|
||||
locations."= /world-of-sosal/" = {
|
||||
extraConfig = ''
|
||||
return 302 /world-of-sosal/index.html;
|
||||
'';
|
||||
};
|
||||
locations."= /world-of-sosal/index.html" = {
|
||||
extraConfig = ''
|
||||
alias ${./static/world-of-sosal/index.html};
|
||||
default_type text/html;
|
||||
add_header Cache-Control "no-cache" always;
|
||||
limit_except GET {
|
||||
deny all;
|
||||
}
|
||||
'';
|
||||
};
|
||||
locations."= /world-of-sosal/latest.mrpack" = {
|
||||
extraConfig = ''
|
||||
root /var/www/store;
|
||||
default_type application/zip;
|
||||
add_header Content-Disposition "attachment" always;
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate" always;
|
||||
try_files $uri =404;
|
||||
if ($request_method != GET) { return 405; }
|
||||
'';
|
||||
};
|
||||
locations."= /world-of-sosal/SHA256SUMS" = {
|
||||
extraConfig = ''
|
||||
root /var/www/store;
|
||||
default_type text/plain;
|
||||
add_header Content-Disposition "attachment" always;
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate" always;
|
||||
try_files $uri =404;
|
||||
if ($request_method != GET) { return 405; }
|
||||
'';
|
||||
};
|
||||
locations."= /world-of-sosal/releases/" = {
|
||||
extraConfig = ''
|
||||
return 404;
|
||||
'';
|
||||
};
|
||||
locations."~ ^/world-of-sosal/releases/[A-Za-z0-9][A-Za-z0-9._-]*\\.mrpack$" = {
|
||||
extraConfig = ''
|
||||
root /var/www/store;
|
||||
default_type application/zip;
|
||||
add_header Content-Disposition "attachment" always;
|
||||
add_header Cache-Control "public, max-age=31536000, immutable" always;
|
||||
try_files $uri =404;
|
||||
if ($request_method != GET) { return 405; }
|
||||
'';
|
||||
};
|
||||
locations."/world-of-sosal/" = {
|
||||
extraConfig = ''
|
||||
autoindex off;
|
||||
limit_except GET {
|
||||
deny all;
|
||||
}
|
||||
return 404;
|
||||
'';
|
||||
};
|
||||
};
|
||||
virtualHosts."lessons.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
};
|
||||
virtualHosts."snuff.${domain}" = {
|
||||
enableACME = true;
|
||||
@@ -346,6 +534,9 @@ in {
|
||||
extraConfig = ''
|
||||
proxy_pass http://127.0.0.1:11011/;
|
||||
proxy_redirect off;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
'';
|
||||
};
|
||||
};
|
||||
@@ -355,15 +546,21 @@ in {
|
||||
gitea = {
|
||||
enable = true;
|
||||
package = pkgs.hectic.gitea-heatmap;
|
||||
# Keep LFS storage limited to accounts provisioned by administrators.
|
||||
settings.service.DISABLE_REGISTRATION = true;
|
||||
settings.session.COOKIE_SECURE = true;
|
||||
settings.actions.ENABLED = true;
|
||||
# Long CUDA builds must not hit Gitea's default three-hour task watchdog.
|
||||
settings.actions.ENDLESS_TASK_TIMEOUT = "8h";
|
||||
settings.server = {
|
||||
HTTP_ADDR = "127.0.0.1";
|
||||
HTTP_PORT = 11011;
|
||||
ROOT_URL = "https://gitea.${domain}/";
|
||||
SSH_PORT = sshPort;
|
||||
SSH_DOMAIN = "hectic-lab.com";
|
||||
};
|
||||
lfs.enable = true;
|
||||
settings.lfs.LFS_MAX_FILE_SIZE = 536870912;
|
||||
database = {
|
||||
createDatabase = true;
|
||||
type = "postgres";
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
{ domain, ... }:
|
||||
{
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
hectic.services.immich = {
|
||||
enable = true;
|
||||
domain = "immich.${domain}";
|
||||
|
||||
storageBox = {
|
||||
enable = true;
|
||||
credentialsFile = config.sops.secrets."immich/storage-box".path;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{ ... }:
|
||||
{
|
||||
imports = [ (import ../../module/generic/minecraft-public-relay.nix { }) ];
|
||||
services.minecraft-public-relay = {
|
||||
enable = true;
|
||||
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNWWegOVTOF3EOmam32iP7sMybULMTxsXuC+cEGITQ8 minecraft-wow-relay";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>WorldOfSosal</title>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>WorldOfSosal</h1>
|
||||
<p><a href="https://store.hectic-lab.com/minecraft/world-of-sosal/">Install in Prism with automatic mod updates</a></p>
|
||||
<p><a href="https://store.hectic-lab.com/world-of-sosal/latest.mrpack">Download latest pack</a></p>
|
||||
<p><a href="prismlauncher://import?url=https%3A%2F%2Fstore.hectic-lab.com%2Fworld-of-sosal%2Flatest.mrpack">Import latest pack in Prism Launcher</a></p>
|
||||
<p><a href="https://store.hectic-lab.com/world-of-sosal/SHA256SUMS">SHA-256 checksums</a></p>
|
||||
<p>Updates are manual. Packs imported from arbitrary URLs do not update automatically.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,52 @@
|
||||
# WorldOfSosal pack publishing
|
||||
|
||||
The public endpoint is `https://store.hectic-lab.com/world-of-sosal/`. Nix
|
||||
deploys only its landing page and nginx configuration. Pack files, the checksum
|
||||
manifest, and `latest.mrpack` stay under `/var/www/store/world-of-sosal` on the
|
||||
host and never enter Git or the Nix store.
|
||||
|
||||
## Publish an uploaded pack
|
||||
|
||||
Run these commands on `hectic-lab` as root after the Storage Box pack has
|
||||
already been uploaded to a local staging path. Pick a stable version name; do
|
||||
not replace an existing versioned release.
|
||||
|
||||
```sh
|
||||
set -eu
|
||||
source_pack=/path/to/already-uploaded/WorldOfSosal.mrpack
|
||||
version=2026-09-16
|
||||
root=/var/www/store/world-of-sosal
|
||||
release_name="WorldOfSosal-${version}.mrpack"
|
||||
release_path="$root/releases/$release_name"
|
||||
|
||||
printf '%s %s\n' \
|
||||
f8c18acb9208e4592725632ae50dab4f9c308483b34fd43a6507c74fdbf8169f \
|
||||
"$source_pack" | sha256sum --check --status
|
||||
test ! -e "$release_path"
|
||||
install -o root -g nginx -m 0640 "$source_pack" "$release_path.new"
|
||||
mv -T "$release_path.new" "$release_path"
|
||||
|
||||
manifest="$root/.SHA256SUMS.$$"
|
||||
(cd "$root/releases" && sha256sum -- *.mrpack) > "$manifest"
|
||||
chown root:nginx "$manifest"
|
||||
chmod 0640 "$manifest"
|
||||
mv -Tf "$manifest" "$root/SHA256SUMS"
|
||||
|
||||
latest="$root/.latest.mrpack.$$"
|
||||
ln -s "releases/$release_name" "$latest"
|
||||
mv -Tf "$latest" "$root/latest.mrpack"
|
||||
```
|
||||
|
||||
Versioned releases use a one-year immutable cache policy. `latest.mrpack` and
|
||||
`SHA256SUMS` disable caching so an atomic replacement becomes visible quickly.
|
||||
The manifest is available at
|
||||
`https://store.hectic-lab.com/world-of-sosal/SHA256SUMS`.
|
||||
|
||||
Import the current pack in Prism Launcher with:
|
||||
|
||||
```text
|
||||
prismlauncher://import?url=https%3A%2F%2Fstore.hectic-lab.com%2Fworld-of-sosal%2Flatest.mrpack
|
||||
```
|
||||
|
||||
Direct URL imports do not auto-update. Repeat the publication and import steps
|
||||
for each new pack version.
|
||||
@@ -1,3 +1,17 @@
|
||||
# Current Minecraft access (2026-09-18)
|
||||
|
||||
WorldOfSosal on the WoW map uses `store.hectic-lab.com:25568` publicly.
|
||||
The game server is `neuro:25567`; a restricted persistent reverse SSH tunnel
|
||||
connects it to the public relay. See `docs/minecraft-prism.md` and the Nix modules
|
||||
`minecraft/public-tunnel.nix` / `hectic-lab/minecraft-wow-proxy.nix`.
|
||||
|
||||
Verified LAN: neuro is `192.168.88.10`, gateway `192.168.88.1`.
|
||||
SSH access is `95.31.254.84:34457`. Direct external Minecraft TCP probes timed out,
|
||||
and no UPnP IGD was discovered. The old TP-Link network and manual port-forward
|
||||
instructions below describe the previous network, not the active configuration.
|
||||
|
||||
---
|
||||
|
||||
# Router Access (TP-Link)
|
||||
|
||||
The server `neuro` is behind a NAT router at `192.168.0.1`.
|
||||
@@ -50,6 +64,8 @@ Ports that need to be forwarded from router to `192.168.0.10`:
|
||||
| 5269 | 5269 | TCP | XMPP (s2s) |
|
||||
| 10000 | 10000 | UDP | Jitsi Videobridge |
|
||||
| 25565 | 25565 | TCP | Minecraft |
|
||||
| 25567 | 25567 | TCP | Minecraft WoW Mine map |
|
||||
| 25568 | 25568 | TCP | Minecraft World of Sosal |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
||||
@@ -17,10 +17,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
|
||||
];
|
||||
config.allowUnfreePredicate = pkg:
|
||||
self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
||||
"minecraft-server"
|
||||
"neoforge"
|
||||
"minecraft-server"
|
||||
"neoforge"
|
||||
"steamcmd"
|
||||
"steam-unwrapped"
|
||||
|
||||
"nvidia-x11"
|
||||
"nvidia-x11"
|
||||
];
|
||||
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
|
||||
config.permittedInsecurePackages = [
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
{
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
services.minecraft-servers = {
|
||||
enable = true;
|
||||
eula = true;
|
||||
openFirewall = true;
|
||||
|
||||
servers = {
|
||||
vanilla = {
|
||||
enable = true;
|
||||
jvmOpts = "-Xmx6G -Xms2G";
|
||||
package = pkgs.minecraftServers.vanilla-1_21_11;
|
||||
|
||||
serverProperties = {
|
||||
server-port = 25565;
|
||||
difficulty = "hard";
|
||||
online-mode = true;
|
||||
view-distance = 32;
|
||||
level-seed = "8306359138650378643";
|
||||
pause-when-empty-seconds = 0;
|
||||
};
|
||||
};
|
||||
|
||||
createAeronautics = {
|
||||
enable = true;
|
||||
jvmOpts = "-Xmx8G -Xms2G";
|
||||
package = pkgs.minecraftServers.neoforge-1_21_1;
|
||||
|
||||
symlinks = {
|
||||
mods = pkgs.linkFarmFromDrvs "mods" (
|
||||
builtins.attrValues {
|
||||
Sable = pkgs.fetchurl {
|
||||
url = "https://cdn.modrinth.com/data/T9PomCSv/versions/g8CObHcP/sable-neoforge-1.21.1-1.1.3.jar";
|
||||
sha512 = "8180e214681c171c9e3b7fa307f7a92bd7de0b8125d671291425f04a4ba26b408758d8ea80a6386d8e73bb1e6b02caf3f20afb9b91ecedd48c37ed44363ac961";
|
||||
};
|
||||
Create = pkgs.fetchurl {
|
||||
url = "https://cdn.modrinth.com/data/LNytGWDc/versions/UjX6dr61/create-1.21.1-6.0.10.jar";
|
||||
sha512 = "11cc8fc049d2f67f6548c7abfada6b82a3adb5c7ca410a742de04bbca76e03862c518721b88d806f6e6d768a4d68531fdb903a85859b25d1484d550cc7bafd4b";
|
||||
};
|
||||
CreateAeronautics = pkgs.fetchurl {
|
||||
url = "https://cdn.modrinth.com/data/oWaK0Q19/versions/1sv6OtSz/create-aeronautics-bundled-1.21.1-1.1.3.jar";
|
||||
sha512 = "94831bc4702b3864524258fa0a73a50ab3cd37e9c157b5c6688a6845b866ec5838452804050b55e490549d91dad909fc37f0d619f354c5676e2e2651b9c15ec6";
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
serverProperties = {
|
||||
server-port = 25566;
|
||||
difficulty = "hard";
|
||||
online-mode = true;
|
||||
view-distance = 20;
|
||||
pause-when-empty-seconds = 0;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{ pkgs, ... }:
|
||||
|
||||
{
|
||||
services.minecraft-servers.servers.createAeronautics = {
|
||||
enable = true;
|
||||
jvmOpts = "-Xmx8G -Xms2G";
|
||||
package = pkgs.minecraftServers.neoforge-1_21_1;
|
||||
|
||||
symlinks = {
|
||||
mods = import ./mods.nix { inherit pkgs; };
|
||||
};
|
||||
|
||||
serverProperties = {
|
||||
server-port = 25566;
|
||||
difficulty = "hard";
|
||||
online-mode = true;
|
||||
view-distance = 20;
|
||||
pause-when-empty-seconds = 0;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
imports = [
|
||||
./vanilla.nix
|
||||
./create-aeronautics.nix
|
||||
./wow-mine-map.nix
|
||||
./world-of-sosal.nix
|
||||
./public-tunnel.nix
|
||||
./world-import.nix
|
||||
./modpack-import.nix
|
||||
];
|
||||
|
||||
services.minecraft-servers = {
|
||||
enable = true;
|
||||
eula = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) mkOption types;
|
||||
|
||||
cfg = config.services.minecraft-modpack-imports;
|
||||
enabledImports = lib.filterAttrs (_: import: import.enable) cfg;
|
||||
dataDir = config.services.minecraft-servers.dataDir;
|
||||
minecraftServers = config.services.minecraft-servers.servers;
|
||||
targetServers = lib.mapAttrsToList (_: import: import.serverName) enabledImports;
|
||||
|
||||
importerUser = name: let
|
||||
descriptiveName = "mc-pack-${name}";
|
||||
in
|
||||
if builtins.stringLength descriptiveName <= 31
|
||||
then descriptiveName
|
||||
else "mc-pack-${builtins.substring 0 16 (builtins.hashString "sha256" name)}";
|
||||
|
||||
stateDirectory = cacheDir:
|
||||
if lib.hasPrefix "/var/lib/" cacheDir
|
||||
then lib.removePrefix "/var/lib/" cacheDir
|
||||
else null;
|
||||
|
||||
stateDirectoryCompatible = cacheDir: let
|
||||
relative = stateDirectory cacheDir;
|
||||
components = lib.splitString "/" (if relative == null then "" else relative);
|
||||
in
|
||||
relative != null
|
||||
&& relative != ""
|
||||
&& lib.all (component: component != "" && component != "." && component != "..") components;
|
||||
|
||||
escapeSftp = value:
|
||||
"\"${lib.replaceStrings ["\\" "\""] ["\\\\" "\\\""] value}\"";
|
||||
|
||||
escapeTmpfiles = value:
|
||||
lib.replaceStrings ["%" " " "\t"] ["%%" "\\x20" "\\x09"] value;
|
||||
|
||||
importerServices = lib.mkMerge (lib.mapAttrsToList (name: import: let
|
||||
user = importerUser name;
|
||||
unitName = "minecraft-modpack-import-${name}";
|
||||
serverUnit = "minecraft-server-${import.serverName}.service";
|
||||
cacheStateDirectory = stateDirectory import.cacheDir;
|
||||
serverDir = "${dataDir}/${import.serverName}";
|
||||
in {
|
||||
${unitName} = {
|
||||
description = "Import Minecraft Modrinth pack ${name}";
|
||||
before = [ serverUnit ];
|
||||
wants = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.findutils
|
||||
pkgs.jq
|
||||
pkgs.openssh
|
||||
pkgs.unar
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = user;
|
||||
Group = user;
|
||||
RemainAfterExit = true;
|
||||
TimeoutStartSec = import.timeout;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectControlGroups = true;
|
||||
NoNewPrivileges = true;
|
||||
RestrictSUIDSGID = true;
|
||||
LockPersonality = true;
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
ReadWritePaths = [ import.cacheDir serverDir ];
|
||||
UMask = "0007";
|
||||
} // lib.optionalAttrs (stateDirectoryCompatible import.cacheDir) {
|
||||
StateDirectory = cacheStateDirectory;
|
||||
};
|
||||
script = ''
|
||||
set -eu
|
||||
umask 007
|
||||
|
||||
cache_dir=${lib.escapeShellArg import.cacheDir}
|
||||
server_dir=${lib.escapeShellArg serverDir}
|
||||
archive_name=${lib.escapeShellArg import.archiveName}
|
||||
archive="$cache_dir/$archive_name"
|
||||
temporary_archive="$cache_dir/.$archive_name.$$"
|
||||
extraction_dir="$cache_dir/.extract-${name}.$$"
|
||||
staging_dir="$cache_dir/.stage-${name}.$$"
|
||||
managed_paths="$cache_dir/managed-paths"
|
||||
new_managed_paths="$cache_dir/.managed-paths.$$"
|
||||
key=${lib.escapeShellArg import.sshKeyFile}
|
||||
|
||||
cleanup() {
|
||||
rm -f "$temporary_archive" "$new_managed_paths"
|
||||
rm -rf "$extraction_dir" "$staging_dir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
safe_relative_path() {
|
||||
case "$1" in
|
||||
""|/*|*\\*|.|..|./*|../*|*/./*|*/../*|*/.|*/..)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
archive_valid() {
|
||||
[ -f "$archive" ] && printf '%s %s\n' \
|
||||
${lib.escapeShellArg import.archiveSha256} \
|
||||
"$archive" | sha256sum -c --status
|
||||
}
|
||||
|
||||
archive_entries_valid() {
|
||||
lsar -json "$archive" | jq -e '
|
||||
(.lsarContents // .entries) as $entries
|
||||
| ($entries | type == "array")
|
||||
and ($entries | all(.[];
|
||||
(.XADFileName // .XADPath) as $path
|
||||
| ($path | type == "string")
|
||||
and ($path | startswith("/") | not)
|
||||
and ($path | contains("\\") | not)
|
||||
and ($path | test("[[:cntrl:]]") | not)
|
||||
and ([$path | split("/")[] | select(. == "" or . == "." or . == "..")] | length == 0)
|
||||
and ((.XADIsSymbolicLink // false) | not)
|
||||
and ((.XADIsHardLink // false) | not)
|
||||
and ((.XADIsDevice // false) | not)
|
||||
and ((.XADIsFIFO // false) | not)
|
||||
and ((.XADIsSocket // false) | not)
|
||||
)
|
||||
)
|
||||
' >/dev/null
|
||||
}
|
||||
|
||||
mkdir -p "$cache_dir" "$server_dir"
|
||||
chmod 0700 "$cache_dir"
|
||||
|
||||
if ! archive_valid; then
|
||||
rm -f "$archive"
|
||||
downloaded=false
|
||||
attempt=1
|
||||
while [ "$attempt" -le ${toString import.retries} ]; do
|
||||
rm -f "$temporary_archive"
|
||||
if sftp \
|
||||
-o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=yes \
|
||||
-o UserKnownHostsFile=/etc/ssh/ssh_known_hosts \
|
||||
-i "$key" \
|
||||
-b - \
|
||||
${lib.escapeShellArg "${import.remoteUser}@${import.remoteHost}"} <<EOF
|
||||
get ${escapeSftp import.remotePath} "$temporary_archive"
|
||||
EOF
|
||||
then
|
||||
if printf '%s %s\n' \
|
||||
${lib.escapeShellArg import.archiveSha256} \
|
||||
"$temporary_archive" | sha256sum -c --status; then
|
||||
mv "$temporary_archive" "$archive"
|
||||
downloaded=true
|
||||
break
|
||||
fi
|
||||
fi
|
||||
rm -f "$temporary_archive"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
if [ "$downloaded" != true ]; then
|
||||
echo "Unable to download verified Minecraft modpack ${name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! archive_entries_valid; then
|
||||
echo "Modpack archive contains unsafe entries" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$extraction_dir" "$staging_dir"
|
||||
unar -quiet -output-directory "$extraction_dir" "$archive"
|
||||
|
||||
find "$extraction_dir" \
|
||||
\( -type l -o -type b -o -type c -o -type p -o -type s \) \
|
||||
-delete
|
||||
|
||||
manifest=$(find "$extraction_dir" -type f -name modrinth.index.json -print)
|
||||
if [ -z "$manifest" ] || [ "$(printf '%s\n' "$manifest" | wc -l)" -ne 1 ]; then
|
||||
echo "Modpack must contain exactly one modrinth.index.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
pack_root=$(dirname "$manifest")
|
||||
|
||||
if ! jq -e \
|
||||
--argjson expectedDependencies ${lib.escapeShellArg (builtins.toJSON import.expectedDependencies)} \
|
||||
'
|
||||
. as $manifest
|
||||
| .formatVersion == 1
|
||||
and ($expectedDependencies | to_entries | all(.[];
|
||||
$manifest.dependencies[.key] == .value
|
||||
))
|
||||
and (.files | type == "array")
|
||||
and all(.files[];
|
||||
((.env.server // "required") == "unsupported")
|
||||
or (
|
||||
(.path | type == "string")
|
||||
and (.path | length > 0)
|
||||
and (.path | startswith("mods/"))
|
||||
and (.path | startswith("/") | not)
|
||||
and (.path | contains("\\") | not)
|
||||
and (.path | test("[[:cntrl:]]") | not)
|
||||
and ([.path | split("/")[] | select(. == "" or . == "." or . == "..")] | length == 0)
|
||||
and (.hashes.sha512 | type == "string")
|
||||
and (.hashes.sha512 | test("^[0-9a-fA-F]{128}$"))
|
||||
and (.downloads | type == "array")
|
||||
and (.downloads | length > 0)
|
||||
and (.downloads[0] | type == "string")
|
||||
and (.downloads[0] | startswith("https://"))
|
||||
and (.downloads[0] | test("[[:cntrl:]]") | not)
|
||||
)
|
||||
)
|
||||
' "$manifest" >/dev/null; then
|
||||
echo "Modpack manifest contains unsafe or invalid server files" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: > "$new_managed_paths"
|
||||
jq -r '
|
||||
.files[]
|
||||
| select((.env.server // "required") != "unsupported")
|
||||
| [.path, .hashes.sha512, .downloads[0]]
|
||||
| @tsv
|
||||
' "$manifest" |
|
||||
while IFS="$(printf '\t')" read -r relative expected_hash url; do
|
||||
safe_relative_path "$relative" || exit 1
|
||||
destination="$staging_dir/$relative"
|
||||
mkdir -p "$(dirname "$destination")"
|
||||
curl --fail --location --silent --show-error \
|
||||
--retry ${toString import.retries} \
|
||||
--output "$destination" \
|
||||
"$url"
|
||||
if ! printf '%s %s\n' "$expected_hash" "$destination" |
|
||||
sha512sum -c --status; then
|
||||
echo "SHA-512 mismatch for $relative" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$relative" >> "$new_managed_paths"
|
||||
done
|
||||
|
||||
overrides_dir="$pack_root/overrides"
|
||||
if [ -d "$overrides_dir" ]; then
|
||||
find "$overrides_dir" \
|
||||
\( -type l -o -type b -o -type c -o -type p -o -type s \) \
|
||||
-delete
|
||||
cp -R "$overrides_dir/." "$staging_dir/"
|
||||
find "$overrides_dir" -type f -printf '%P\n' |
|
||||
while IFS= read -r relative; do
|
||||
safe_relative_path "$relative" || exit 1
|
||||
printf '%s\n' "$relative"
|
||||
done >> "$new_managed_paths"
|
||||
fi
|
||||
|
||||
# Nix Minecraft manages eula.txt via a symlink. Only reject symlinks
|
||||
# in destinations we actually touch, including their parent directories.
|
||||
safe_target_path() {
|
||||
safe_relative_path "$1" || return 1
|
||||
target="$server_dir/$1"
|
||||
while [ "$target" != "$server_dir" ]; do
|
||||
if [ -L "$target" ]; then
|
||||
echo "Modpack destination contains symlink: $target" >&2
|
||||
return 1
|
||||
fi
|
||||
target=$(dirname "$target")
|
||||
done
|
||||
}
|
||||
while IFS= read -r relative; do
|
||||
safe_target_path "$relative" || exit 1
|
||||
done < "$new_managed_paths"
|
||||
if [ -f "$managed_paths" ]; then
|
||||
while IFS= read -r relative; do
|
||||
safe_target_path "$relative" || exit 1
|
||||
done < "$managed_paths"
|
||||
fi
|
||||
|
||||
while IFS= read -r relative; do
|
||||
safe_relative_path "$relative" || exit 1
|
||||
source_file="$staging_dir/$relative"
|
||||
target_file="$server_dir/$relative"
|
||||
install -d -m 0770 -g minecraft "$(dirname "$target_file")"
|
||||
install -m 0660 -g minecraft "$source_file" "$target_file"
|
||||
done < "$new_managed_paths"
|
||||
|
||||
if [ -f "$managed_paths" ]; then
|
||||
while IFS= read -r old_relative; do
|
||||
safe_relative_path "$old_relative" || {
|
||||
echo "Unsafe path in previous managed-paths file" >&2
|
||||
exit 1
|
||||
}
|
||||
keep=false
|
||||
while IFS= read -r relative; do
|
||||
if [ "$old_relative" = "$relative" ]; then
|
||||
keep=true
|
||||
break
|
||||
fi
|
||||
done < "$new_managed_paths"
|
||||
if [ "$keep" != true ]; then
|
||||
rm -f "$server_dir/$old_relative"
|
||||
fi
|
||||
done < "$managed_paths"
|
||||
fi
|
||||
|
||||
mv "$new_managed_paths" "$managed_paths"
|
||||
'';
|
||||
};
|
||||
|
||||
"minecraft-server-${import.serverName}" = {
|
||||
requires = [ "${unitName}.service" ];
|
||||
after = [ "${unitName}.service" ];
|
||||
};
|
||||
}) enabledImports);
|
||||
in {
|
||||
options.services.minecraft-modpack-imports = mkOption {
|
||||
default = { };
|
||||
type = types.attrsOf (types.submodule ({ name, ... }: {
|
||||
options = {
|
||||
enable = lib.mkEnableOption "Minecraft Modrinth pack import ${name}";
|
||||
|
||||
serverName = mkOption {
|
||||
type = types.str;
|
||||
description = "minecraft-servers server receiving imported pack";
|
||||
};
|
||||
|
||||
remoteHost = mkOption {
|
||||
type = types.str;
|
||||
description = "SSH host serving Modrinth pack archive";
|
||||
};
|
||||
|
||||
remoteUser = mkOption {
|
||||
type = types.str;
|
||||
description = "SSH user used to download Modrinth pack archive";
|
||||
};
|
||||
|
||||
remotePath = mkOption {
|
||||
type = types.str;
|
||||
description = "Remote path to Modrinth pack archive";
|
||||
};
|
||||
|
||||
archiveName = mkOption {
|
||||
type = types.str;
|
||||
description = "Archive file name inside cache directory";
|
||||
};
|
||||
|
||||
cacheDir = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/minecraft-modpacks/${name}";
|
||||
description = "Persistent Modrinth archive and importer state directory";
|
||||
};
|
||||
|
||||
archiveSha256 = mkOption {
|
||||
type = types.strMatching "[0-9a-fA-F]{64}";
|
||||
description = "Expected SHA-256 digest of Modrinth pack archive";
|
||||
};
|
||||
|
||||
expectedDependencies = mkOption {
|
||||
type = types.attrsOf types.str;
|
||||
default = { };
|
||||
description = "Required dependency versions in modrinth.index.json";
|
||||
};
|
||||
|
||||
sshKeyFile = mkOption {
|
||||
type = types.str;
|
||||
description = "Runtime path to private SSH key";
|
||||
};
|
||||
|
||||
hostPublicKey = mkOption {
|
||||
type = types.str;
|
||||
description = "Pinned SSH host public key";
|
||||
};
|
||||
|
||||
retries = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 3;
|
||||
description = "Maximum SFTP attempts and curl retry count";
|
||||
};
|
||||
|
||||
timeout = mkOption {
|
||||
type = types.str;
|
||||
default = "30min";
|
||||
description = "Importer service start timeout";
|
||||
};
|
||||
};
|
||||
}));
|
||||
description = "Modrinth packs imported before selected Minecraft servers start";
|
||||
};
|
||||
|
||||
config = lib.mkIf (enabledImports != { }) {
|
||||
assertions = lib.flatten (lib.mapAttrsToList (name: import: [
|
||||
{
|
||||
assertion = builtins.match "[A-Za-z0-9_-]+" name != null;
|
||||
message = "services.minecraft-modpack-imports.${name}: name must contain only letters, digits, underscores, or hyphens";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "/.*" import.cacheDir != null;
|
||||
message = "services.minecraft-modpack-imports.${name}.cacheDir must be absolute";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "/var/lib(/[A-Za-z0-9][A-Za-z0-9._-]*)+" import.cacheDir != null;
|
||||
message = "services.minecraft-modpack-imports.${name}.cacheDir must be beneath /var/lib with safe path components";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "[A-Za-z0-9_-]+" import.serverName != null;
|
||||
message = "services.minecraft-modpack-imports.${name}.serverName must contain only letters, digits, underscores, or hyphens";
|
||||
}
|
||||
{
|
||||
assertion = !lib.hasInfix "\n" import.remotePath && !lib.hasInfix "\r" import.remotePath;
|
||||
message = "services.minecraft-modpack-imports.${name}.remotePath must not contain newlines";
|
||||
}
|
||||
{
|
||||
assertion = builtins.hasAttr import.serverName minecraftServers
|
||||
&& (builtins.getAttr import.serverName minecraftServers).enable;
|
||||
message = "services.minecraft-modpack-imports.${name}.serverName must name an enabled Minecraft server";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "[A-Za-z0-9._-]+" import.archiveName != null
|
||||
&& import.archiveName != "."
|
||||
&& import.archiveName != ".."
|
||||
&& import.archiveName != "managed-paths";
|
||||
message = "services.minecraft-modpack-imports.${name}.archiveName must be a file name";
|
||||
}
|
||||
{
|
||||
assertion = lib.length (lib.unique targetServers) == lib.length targetServers;
|
||||
message = "services.minecraft-modpack-imports: each server target must be unique";
|
||||
}
|
||||
]) enabledImports);
|
||||
|
||||
users.groups = lib.mapAttrs' (name: _: lib.nameValuePair (importerUser name) { }) enabledImports;
|
||||
users.users = lib.mapAttrs' (name: _: let
|
||||
user = importerUser name;
|
||||
in lib.nameValuePair user {
|
||||
description = "Minecraft modpack importer ${name}";
|
||||
isSystemUser = true;
|
||||
group = user;
|
||||
extraGroups = [ "minecraft" ];
|
||||
}) enabledImports;
|
||||
|
||||
programs.ssh.knownHosts = lib.mapAttrs' (name: import:
|
||||
lib.nameValuePair "minecraft-modpack-import-${name}" {
|
||||
hostNames = [ import.remoteHost ];
|
||||
publicKey = import.hostPublicKey;
|
||||
}) enabledImports;
|
||||
|
||||
systemd.tmpfiles.rules = lib.flatten (lib.mapAttrsToList (name: import:
|
||||
lib.optional (!stateDirectoryCompatible import.cacheDir)
|
||||
"d ${escapeTmpfiles import.cacheDir} 0700 ${importerUser name} ${importerUser name} -") enabledImports);
|
||||
|
||||
systemd.services = importerServices;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ pkgs }:
|
||||
|
||||
pkgs.linkFarmFromDrvs "create-aeronautics-mods" (
|
||||
builtins.attrValues {
|
||||
Sable = pkgs.fetchurl {
|
||||
url = "https://cdn.modrinth.com/data/T9PomCSv/versions/g8CObHcP/sable-neoforge-1.21.1-1.1.3.jar";
|
||||
sha512 = "8180e214681c171c9e3b7fa307f7a92bd7de0b8125d671291425f04a4ba26b408758d8ea80a6386d8e73bb1e6b02caf3f20afb9b91ecedd48c37ed44363ac961";
|
||||
};
|
||||
Create = pkgs.fetchurl {
|
||||
url = "https://cdn.modrinth.com/data/LNytGWDc/versions/UjX6dr61/create-1.21.1-6.0.10.jar";
|
||||
sha512 = "11cc8fc049d2f67f6548c7abfada6b82a3adb5c7ca410a742de04bbca76e03862c518721b88d806f6e6d768a4d68531fdb903a85859b25d1484d550cc7bafd4b";
|
||||
};
|
||||
CreateAeronautics = pkgs.fetchurl {
|
||||
url = "https://cdn.modrinth.com/data/oWaK0Q19/versions/1sv6OtSz/create-aeronautics-bundled-1.21.1-1.1.3.jar";
|
||||
sha512 = "94831bc4702b3864524258fa0a73a50ab3cd37e9c157b5c6688a6845b866ec5838452804050b55e490549d91dad909fc37f0d619f354c5676e2e2651b9c15ec6";
|
||||
};
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,363 @@
|
||||
{
|
||||
"build": {
|
||||
"version": "21.1.250",
|
||||
"src": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/neoforge/21.1.250/neoforge-21.1.250-installer.jar",
|
||||
"hash": "sha256-DkepG6ITmo20v3Ynrwgfe1eJtQi7A57o3qEnK3lpPWA="
|
||||
},
|
||||
"libraries": [
|
||||
"net.neoforged.fancymodloader:earlydisplay:4.0.44",
|
||||
"net.neoforged.fancymodloader:loader:4.0.44",
|
||||
"net.neoforged.accesstransformers:at-modlauncher:10.0.1",
|
||||
"net.neoforged:accesstransformers:10.0.1",
|
||||
"net.neoforged:bus:8.0.5",
|
||||
"net.neoforged:coremods:7.0.3",
|
||||
"cpw.mods:modlauncher:11.0.5",
|
||||
"net.neoforged:mergetool:2.0.0:api",
|
||||
"com.electronwill.night-config:toml:3.8.3",
|
||||
"com.electronwill.night-config:core:3.8.3",
|
||||
"net.neoforged:JarJarSelector:0.4.1",
|
||||
"net.neoforged:JarJarMetadata:0.4.1",
|
||||
"org.apache.maven:maven-artifact:3.8.5",
|
||||
"net.jodah:typetools:0.6.3",
|
||||
"net.minecrell:terminalconsoleappender:1.3.0",
|
||||
"net.fabricmc:sponge-mixin:0.15.2+mixin.0.8.7",
|
||||
"org.openjdk.nashorn:nashorn-core:15.4",
|
||||
"org.apache.commons:commons-lang3:3.14.0",
|
||||
"cpw.mods:bootstraplauncher:2.0.2",
|
||||
"cpw.mods:securejarhandler:3.0.8",
|
||||
"org.ow2.asm:asm-commons:9.10.1",
|
||||
"org.ow2.asm:asm-util:9.10.1",
|
||||
"org.ow2.asm:asm-analysis:9.10.1",
|
||||
"org.ow2.asm:asm-tree:9.10.1",
|
||||
"org.ow2.asm:asm:9.10.1",
|
||||
"net.neoforged:JarJarFileSystems:0.4.1",
|
||||
"net.sf.jopt-simple:jopt-simple:5.0.4",
|
||||
"org.slf4j:slf4j-api:2.0.9",
|
||||
"org.antlr:antlr4-runtime:4.13.1",
|
||||
"com.mojang:logging:1.2.7",
|
||||
"org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1",
|
||||
"org.apache.logging.log4j:log4j-core:2.22.1",
|
||||
"org.apache.logging.log4j:log4j-api:2.22.1",
|
||||
"org.jline:jline-reader:3.20.0",
|
||||
"org.jline:jline-terminal:3.20.0",
|
||||
"commons-io:commons-io:2.15.1",
|
||||
"net.minecraftforge:srgutils:0.4.15",
|
||||
"com.google.guava:guava:32.1.2-jre",
|
||||
"com.google.guava:failureaccess:1.0.1",
|
||||
"com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava",
|
||||
"com.google.code.findbugs:jsr305:3.0.2",
|
||||
"org.checkerframework:checker-qual:3.33.0",
|
||||
"com.google.errorprone:error_prone_annotations:2.18.0",
|
||||
"com.google.j2objc:j2objc-annotations:2.8",
|
||||
"com.google.code.gson:gson:2.10.1",
|
||||
"org.codehaus.plexus:plexus-utils:3.3.0",
|
||||
"com.machinezoo.noexception:noexception:1.7.1",
|
||||
"net.neoforged:neoform:1.21.1-20240808.144430@zip",
|
||||
"net.neoforged.installertools:binarypatcher:2.1.2:fatjar",
|
||||
"net.neoforged:AutoRenamingTool:2.0.3:all",
|
||||
"net.neoforged.installertools:installertools:2.1.2",
|
||||
"net.neoforged:srgutils:1.0.0",
|
||||
"net.md-5:SpecialSource:1.11.0",
|
||||
"com.google.code.gson:gson:2.8.9",
|
||||
"de.siegmar:fastcsv:2.0.0",
|
||||
"org.ow2.asm:asm-commons:9.3",
|
||||
"net.neoforged.installertools:cli-utils:2.1.2",
|
||||
"com.google.guava:guava:20.0",
|
||||
"com.opencsv:opencsv:4.4",
|
||||
"org.ow2.asm:asm-analysis:9.3",
|
||||
"org.ow2.asm:asm-tree:9.3",
|
||||
"org.ow2.asm:asm:9.3",
|
||||
"org.apache.commons:commons-text:1.3",
|
||||
"org.apache.commons:commons-lang3:3.8.1",
|
||||
"commons-beanutils:commons-beanutils:1.9.3",
|
||||
"org.apache.commons:commons-collections4:4.2",
|
||||
"commons-logging:commons-logging:1.2",
|
||||
"commons-collections:commons-collections:3.2.2",
|
||||
"net.neoforged.installertools:jarsplitter:2.1.2",
|
||||
"net.neoforged:neoforge:21.1.250:universal"
|
||||
]
|
||||
},
|
||||
"libraryLocks": {
|
||||
"net.neoforged.fancymodloader:earlydisplay:4.0.44": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/fancymodloader/earlydisplay/4.0.44/earlydisplay-4.0.44.jar",
|
||||
"hash": "sha1-yyr88VZyGA7KHEmxybEgiVHvMi4="
|
||||
},
|
||||
"net.neoforged.fancymodloader:loader:4.0.44": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/fancymodloader/loader/4.0.44/loader-4.0.44.jar",
|
||||
"hash": "sha1-Y0M2CLaDAqRC49uoeIDlT0LqBYM="
|
||||
},
|
||||
"net.neoforged.accesstransformers:at-modlauncher:10.0.1": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/accesstransformers/at-modlauncher/10.0.1/at-modlauncher-10.0.1.jar",
|
||||
"hash": "sha1-WrpQICrO6tCGvAn7wnUcnwXLSJA="
|
||||
},
|
||||
"net.neoforged:accesstransformers:10.0.1": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/accesstransformers/10.0.1/accesstransformers-10.0.1.jar",
|
||||
"hash": "sha1-/YO1cl926ukRXpNV+hxFampEFAA="
|
||||
},
|
||||
"net.neoforged:bus:8.0.5": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/bus/8.0.5/bus-8.0.5.jar",
|
||||
"hash": "sha1-Wy0zKFq10VVOl5itmMQNbqOGi9U="
|
||||
},
|
||||
"net.neoforged:coremods:7.0.3": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/coremods/7.0.3/coremods-7.0.3.jar",
|
||||
"hash": "sha1-CRR+b2OLQnKzvV/I+SrTeAJRLGw="
|
||||
},
|
||||
"cpw.mods:modlauncher:11.0.5": {
|
||||
"url": "https://maven.neoforged.net/releases/cpw/mods/modlauncher/11.0.5/modlauncher-11.0.5.jar",
|
||||
"hash": "sha1-uPDUkpT3M/22FzkxsmNVPpQ9yVA="
|
||||
},
|
||||
"net.neoforged:mergetool:2.0.0:api": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/mergetool/2.0.0/mergetool-2.0.0-api.jar",
|
||||
"hash": "sha1-Uv4ZSb5k4zA6q6qiHjFfVR25yfQ="
|
||||
},
|
||||
"com.electronwill.night-config:toml:3.8.3": {
|
||||
"url": "https://maven.neoforged.net/releases/com/electronwill/night-config/toml/3.8.3/toml-3.8.3.jar",
|
||||
"hash": "sha1-kLL9bvy0p9W5gQz3j52CSZTXF/I="
|
||||
},
|
||||
"com.electronwill.night-config:core:3.8.3": {
|
||||
"url": "https://maven.neoforged.net/releases/com/electronwill/night-config/core/3.8.3/core-3.8.3.jar",
|
||||
"hash": "sha1-tEKpXwnjSZJ/WpRey1lEVYcPz08="
|
||||
},
|
||||
"net.neoforged:JarJarSelector:0.4.1": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/JarJarSelector/0.4.1/JarJarSelector-0.4.1.jar",
|
||||
"hash": "sha1-+zzHpYryKtKICtuYr21RgSjEfa4="
|
||||
},
|
||||
"net.neoforged:JarJarMetadata:0.4.1": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/JarJarMetadata/0.4.1/JarJarMetadata-0.4.1.jar",
|
||||
"hash": "sha1-+NoDaD3IFpRVbcPhd8Xju3eub8s="
|
||||
},
|
||||
"org.apache.maven:maven-artifact:3.8.5": {
|
||||
"url": "https://maven.neoforged.net/releases/org/apache/maven/maven-artifact/3.8.5/maven-artifact-3.8.5.jar",
|
||||
"hash": "sha1-RDP1DAfevvrtBVO9AGj09I1EkxM="
|
||||
},
|
||||
"net.jodah:typetools:0.6.3": {
|
||||
"url": "https://maven.neoforged.net/releases/net/jodah/typetools/0.6.3/typetools-0.6.3.jar",
|
||||
"hash": "sha1-oBqqbdrqnsB+xPIJSHt6RqUmKDo="
|
||||
},
|
||||
"net.minecrell:terminalconsoleappender:1.3.0": {
|
||||
"url": "https://maven.neoforged.net/releases/net/minecrell/terminalconsoleappender/1.3.0/terminalconsoleappender-1.3.0.jar",
|
||||
"hash": "sha1-tWLpu2EjXJUg4mKCze5x+PgC0fw="
|
||||
},
|
||||
"net.fabricmc:sponge-mixin:0.15.2+mixin.0.8.7": {
|
||||
"url": "https://maven.neoforged.net/releases/net/fabricmc/sponge-mixin/0.15.2+mixin.0.8.7/sponge-mixin-0.15.2+mixin.0.8.7.jar",
|
||||
"hash": "sha1-KvLwIdjgKgIg3CenpytGZtZtRMo="
|
||||
},
|
||||
"org.openjdk.nashorn:nashorn-core:15.4": {
|
||||
"url": "https://maven.neoforged.net/releases/org/openjdk/nashorn/nashorn-core/15.4/nashorn-core-15.4.jar",
|
||||
"hash": "sha1-9n9f+qX1Ewz2+5sTPaAMffO1MqU="
|
||||
},
|
||||
"org.apache.commons:commons-lang3:3.14.0": {
|
||||
"url": "https://libraries.minecraft.net/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar",
|
||||
"hash": "sha1-HtRxGUsC8sbLc0oM1vbxB8Zzr64="
|
||||
},
|
||||
"cpw.mods:bootstraplauncher:2.0.2": {
|
||||
"url": "https://maven.neoforged.net/releases/cpw/mods/bootstraplauncher/2.0.2/bootstraplauncher-2.0.2.jar",
|
||||
"hash": "sha1-Gi0HbLwzsFIMus1ZEiRCeyogBH0="
|
||||
},
|
||||
"cpw.mods:securejarhandler:3.0.8": {
|
||||
"url": "https://maven.neoforged.net/releases/cpw/mods/securejarhandler/3.0.8/securejarhandler-3.0.8.jar",
|
||||
"hash": "sha1-wO+Vzs2GmaBEkFOsfZwWB0jZAs0="
|
||||
},
|
||||
"org.ow2.asm:asm-commons:9.10.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-commons/9.10.1/asm-commons-9.10.1.jar",
|
||||
"hash": "sha1-QinkxV/Y4Bwj+f6YhAdcxiiqzFA="
|
||||
},
|
||||
"org.ow2.asm:asm-util:9.10.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-util/9.10.1/asm-util-9.10.1.jar",
|
||||
"hash": "sha1-e7nUUOjUy/n54ECWxEu/5/uoCxU="
|
||||
},
|
||||
"org.ow2.asm:asm-analysis:9.10.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-analysis/9.10.1/asm-analysis-9.10.1.jar",
|
||||
"hash": "sha1-jUnxTVH2Mssdh8iNHOr1DbDYrxs="
|
||||
},
|
||||
"org.ow2.asm:asm-tree:9.10.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-tree/9.10.1/asm-tree-9.10.1.jar",
|
||||
"hash": "sha1-4kQzKhdWTB0VckSTmahC3jWIG+I="
|
||||
},
|
||||
"org.ow2.asm:asm:9.10.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm/9.10.1/asm-9.10.1.jar",
|
||||
"hash": "sha1-raIUHAzFLuj1xIzV+kzg55TyIjY="
|
||||
},
|
||||
"net.neoforged:JarJarFileSystems:0.4.1": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/JarJarFileSystems/0.4.1/JarJarFileSystems-0.4.1.jar",
|
||||
"hash": "sha1-ePWfid780DLteIsVHKag1ArOeWo="
|
||||
},
|
||||
"net.sf.jopt-simple:jopt-simple:5.0.4": {
|
||||
"url": "https://libraries.minecraft.net/net/sf/jopt-simple/jopt-simple/5.0.4/jopt-simple-5.0.4.jar",
|
||||
"hash": "sha1-T9rC++kt+thqpukwFzb2tDQqP1w="
|
||||
},
|
||||
"org.slf4j:slf4j-api:2.0.9": {
|
||||
"url": "https://libraries.minecraft.net/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar",
|
||||
"hash": "sha1-fPJyb9z7yGEPmnH7PtY5hx8xU0A="
|
||||
},
|
||||
"org.antlr:antlr4-runtime:4.13.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/antlr/antlr4-runtime/4.13.1/antlr4-runtime-4.13.1.jar",
|
||||
"hash": "sha1-FxJbrh2WViTiZe9JVS9kZaK/owc="
|
||||
},
|
||||
"com.mojang:logging:1.2.7": {
|
||||
"url": "https://libraries.minecraft.net/com/mojang/logging/1.2.7/logging-1.2.7.jar",
|
||||
"hash": "sha1-JMuV/7DjQz/W6ETATmgAnlBMocA="
|
||||
},
|
||||
"org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1": {
|
||||
"url": "https://libraries.minecraft.net/org/apache/logging/log4j/log4j-slf4j2-impl/2.22.1/log4j-slf4j2-impl-2.22.1.jar",
|
||||
"hash": "sha1-1+ZpPCYGy35zNQR9e7lt7FLbVmU="
|
||||
},
|
||||
"org.apache.logging.log4j:log4j-core:2.22.1": {
|
||||
"url": "https://libraries.minecraft.net/org/apache/logging/log4j/log4j-core/2.22.1/log4j-core-2.22.1.jar",
|
||||
"hash": "sha1-cYOiVRCgKtAMxqldOz0qfTxajcQ="
|
||||
},
|
||||
"org.apache.logging.log4j:log4j-api:2.22.1": {
|
||||
"url": "https://libraries.minecraft.net/org/apache/logging/log4j/log4j-api/2.22.1/log4j-api-2.22.1.jar",
|
||||
"hash": "sha1-vqb+3mMo+rr9fmg2MWGn6mYFq9E="
|
||||
},
|
||||
"org.jline:jline-reader:3.20.0": {
|
||||
"url": "https://maven.neoforged.net/releases/org/jline/jline-reader/3.20.0/jline-reader-3.20.0.jar",
|
||||
"hash": "sha1-jxVBWwIqJbRz6OFsKK6RMYb/ucQ="
|
||||
},
|
||||
"org.jline:jline-terminal:3.20.0": {
|
||||
"url": "https://maven.neoforged.net/releases/org/jline/jline-terminal/3.20.0/jline-terminal-3.20.0.jar",
|
||||
"hash": "sha1-0N3McI3fUno0VMlBt7kiXMg6Ff8="
|
||||
},
|
||||
"commons-io:commons-io:2.15.1": {
|
||||
"url": "https://libraries.minecraft.net/commons-io/commons-io/2.15.1/commons-io-2.15.1.jar",
|
||||
"hash": "sha1-8RVg2hiatWOlyONRlBQVQw6TBOo="
|
||||
},
|
||||
"net.minecraftforge:srgutils:0.4.15": {
|
||||
"url": "https://maven.neoforged.net/releases/net/minecraftforge/srgutils/0.4.15/srgutils-0.4.15.jar",
|
||||
"hash": "sha1-ykCLExdZR48WTgEPrg1zmX4SX7U="
|
||||
},
|
||||
"com.google.guava:guava:32.1.2-jre": {
|
||||
"url": "https://libraries.minecraft.net/com/google/guava/guava/32.1.2-jre/guava-32.1.2-jre.jar",
|
||||
"hash": "sha1-XmTsfgVkVr7zpLxMb9rvceirYxg="
|
||||
},
|
||||
"com.google.guava:failureaccess:1.0.1": {
|
||||
"url": "https://libraries.minecraft.net/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar",
|
||||
"hash": "sha1-Hc8d44Kgv5Wj2LCElUbIi6wSksk="
|
||||
},
|
||||
"com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava": {
|
||||
"url": "https://libraries.minecraft.net/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar",
|
||||
"hash": "sha1-tCFSbF8pcpWt7xyIblJGw51Kxik="
|
||||
},
|
||||
"com.google.code.findbugs:jsr305:3.0.2": {
|
||||
"url": "https://libraries.minecraft.net/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar",
|
||||
"hash": "sha1-JeouiwwziodzE71GctP+BW6njw0="
|
||||
},
|
||||
"org.checkerframework:checker-qual:3.33.0": {
|
||||
"url": "https://libraries.minecraft.net/org/checkerframework/checker-qual/3.33.0/checker-qual-3.33.0.jar",
|
||||
"hash": "sha1-3itgti2kh2RPwR9zTnPIsLQxI48="
|
||||
},
|
||||
"com.google.errorprone:error_prone_annotations:2.18.0": {
|
||||
"url": "https://libraries.minecraft.net/com/google/errorprone/error_prone_annotations/2.18.0/error_prone_annotations-2.18.0.jar",
|
||||
"hash": "sha1-ibaEJXCW9Uj6Oaffn9qkCdTU35E="
|
||||
},
|
||||
"com.google.j2objc:j2objc-annotations:2.8": {
|
||||
"url": "https://libraries.minecraft.net/com/google/j2objc/j2objc-annotations/2.8/j2objc-annotations-2.8.jar",
|
||||
"hash": "sha1-yFJw4wfnuCLxCGuTaJEkuJdo4nM="
|
||||
},
|
||||
"com.google.code.gson:gson:2.10.1": {
|
||||
"url": "https://libraries.minecraft.net/com/google/code/gson/gson/2.10.1/gson-2.10.1.jar",
|
||||
"hash": "sha1-s63UeNQ4K3jqILFnE5CoWAAv62w="
|
||||
},
|
||||
"org.codehaus.plexus:plexus-utils:3.3.0": {
|
||||
"url": "https://maven.neoforged.net/releases/org/codehaus/plexus/plexus-utils/3.3.0/plexus-utils-3.3.0.jar",
|
||||
"hash": "sha1-z0O1OR3mI7Nv4GaiESe674LGQCI="
|
||||
},
|
||||
"com.machinezoo.noexception:noexception:1.7.1": {
|
||||
"url": "https://maven.neoforged.net/releases/com/machinezoo/noexception/noexception/1.7.1/noexception-1.7.1.jar",
|
||||
"hash": "sha1-tlMwyY44ofkV+lSm5eykllBePwo="
|
||||
},
|
||||
"net.neoforged:neoform:1.21.1-20240808.144430@zip": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/neoform/1.21.1-20240808.144430/neoform-1.21.1-20240808.144430.zip",
|
||||
"hash": "sha1-gR4r2G+izaKBLl6OUdcY6ovW0/Q="
|
||||
},
|
||||
"net.neoforged.installertools:binarypatcher:2.1.2:fatjar": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/installertools/binarypatcher/2.1.2/binarypatcher-2.1.2-fatjar.jar",
|
||||
"hash": "sha1-dZtj7zk+2AQY7B6k0jPNYVLQJjc="
|
||||
},
|
||||
"net.neoforged:AutoRenamingTool:2.0.3:all": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/AutoRenamingTool/2.0.3/AutoRenamingTool-2.0.3-all.jar",
|
||||
"hash": "sha1-2YkMcbQ2b4hsKxAGeCBDpqaBbrY="
|
||||
},
|
||||
"net.neoforged.installertools:installertools:2.1.2": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/installertools/installertools/2.1.2/installertools-2.1.2.jar",
|
||||
"hash": "sha1-clJMA2L4EtiqTNtMA+m0Xitxrjs="
|
||||
},
|
||||
"net.neoforged:srgutils:1.0.0": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/srgutils/1.0.0/srgutils-1.0.0.jar",
|
||||
"hash": "sha1-uf5s2rSUmDIXy8FMxvksjmxhZSY="
|
||||
},
|
||||
"net.md-5:SpecialSource:1.11.0": {
|
||||
"url": "https://maven.neoforged.net/releases/net/md-5/SpecialSource/1.11.0/SpecialSource-1.11.0.jar",
|
||||
"hash": "sha1-Q7hMS7jQHPkKKd/uwclYpLZLr0Y="
|
||||
},
|
||||
"com.google.code.gson:gson:2.8.9": {
|
||||
"url": "https://libraries.minecraft.net/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar",
|
||||
"hash": "sha1-ikMsHWgleB4hoC2y4sM8X94oM7k="
|
||||
},
|
||||
"de.siegmar:fastcsv:2.0.0": {
|
||||
"url": "https://maven.neoforged.net/releases/de/siegmar/fastcsv/2.0.0/fastcsv-2.0.0.jar",
|
||||
"hash": "sha1-thXybAPt6slmYYuTue5PTu1QquE="
|
||||
},
|
||||
"org.ow2.asm:asm-commons:9.3": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-commons/9.3/asm-commons-9.3.jar",
|
||||
"hash": "sha1-HypDLRIS9cNSrmB9e2HcriDCCvU="
|
||||
},
|
||||
"net.neoforged.installertools:cli-utils:2.1.2": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/installertools/cli-utils/2.1.2/cli-utils-2.1.2.jar",
|
||||
"hash": "sha1-5aMRXrnimRF0cmdZRr6KcuSZCtY="
|
||||
},
|
||||
"com.google.guava:guava:20.0": {
|
||||
"url": "https://maven.neoforged.net/releases/com/google/guava/guava/20.0/guava-20.0.jar",
|
||||
"hash": "sha1-iVB3ASSTiOHtXdz4xB9M4b54Me8="
|
||||
},
|
||||
"com.opencsv:opencsv:4.4": {
|
||||
"url": "https://maven.neoforged.net/releases/com/opencsv/opencsv/4.4/opencsv-4.4.jar",
|
||||
"hash": "sha1-Ulkyoe30bJynWqnIPTHcGAwYaGU="
|
||||
},
|
||||
"org.ow2.asm:asm-analysis:9.3": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-analysis/9.3/asm-analysis-9.3.jar",
|
||||
"hash": "sha1-SwcfIRs3w44On1mYVQGXyFk/atg="
|
||||
},
|
||||
"org.ow2.asm:asm-tree:9.3": {
|
||||
"url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-tree/9.3/asm-tree-9.3.jar",
|
||||
"hash": "sha1-eNLs1hMYtaWM0E+yN2NsDoa3fZc="
|
||||
},
|
||||
"org.ow2.asm:asm:9.3": {
|
||||
"url": "https://libraries.minecraft.net/org/ow2/asm/asm/9.3/asm-9.3.jar",
|
||||
"hash": "sha1-jmMA71HB2AGn7WLQfNIhrKOpBkA="
|
||||
},
|
||||
"org.apache.commons:commons-text:1.3": {
|
||||
"url": "https://maven.neoforged.net/releases/org/apache/commons/commons-text/1.3/commons-text-1.3.jar",
|
||||
"hash": "sha1-mr9hcIpmq15V9haaIA2/xYS1Rtk="
|
||||
},
|
||||
"org.apache.commons:commons-lang3:3.8.1": {
|
||||
"url": "https://maven.neoforged.net/releases/org/apache/commons/commons-lang3/3.8.1/commons-lang3-3.8.1.jar",
|
||||
"hash": "sha1-ZQWnKgl9knD3qee/QsQjgoMkd1U="
|
||||
},
|
||||
"commons-beanutils:commons-beanutils:1.9.3": {
|
||||
"url": "https://maven.neoforged.net/releases/commons-beanutils/commons-beanutils/1.9.3/commons-beanutils-1.9.3.jar",
|
||||
"hash": "sha1-yEVwPeM03ca0s80mg1RYyxy6Hz0="
|
||||
},
|
||||
"org.apache.commons:commons-collections4:4.2": {
|
||||
"url": "https://maven.neoforged.net/releases/org/apache/commons/commons-collections4/4.2/commons-collections4-4.2.jar",
|
||||
"hash": "sha1-VOvqCltlPTxoATHnP+gHu494xO0="
|
||||
},
|
||||
"commons-logging:commons-logging:1.2": {
|
||||
"url": "https://libraries.minecraft.net/commons-logging/commons-logging/1.2/commons-logging-1.2.jar",
|
||||
"hash": "sha1-S/wSrf5IQr8HtlfwNpxMtSKVVoY="
|
||||
},
|
||||
"commons-collections:commons-collections:3.2.2": {
|
||||
"url": "https://maven.neoforged.net/releases/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar",
|
||||
"hash": "sha1-itcv45+oyR6q8Sqtsh4MNmH+JtU="
|
||||
},
|
||||
"net.neoforged.installertools:jarsplitter:2.1.2": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/installertools/jarsplitter/2.1.2/jarsplitter-2.1.2.jar",
|
||||
"hash": "sha1-inkWvgoOWJiXvqt8g5ByYxBn5I4="
|
||||
},
|
||||
"net.neoforged:neoforge:21.1.250:universal": {
|
||||
"url": "https://maven.neoforged.net/releases/net/neoforged/neoforge/21.1.250/neoforge-21.1.250-universal.jar",
|
||||
"hash": "sha1-IUxSk8hZmwqq39v6kkU7ojkYohk="
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{ config, pkgs, ... }:
|
||||
let
|
||||
mkTunnel = relay: {
|
||||
description = "WorldOfSosal WoW reverse tunnel to ${relay.name}";
|
||||
startLimitIntervalSec = 0;
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
User = "mc-wow-tunnel";
|
||||
Group = "mc-wow-tunnel";
|
||||
ExecStart = "${pkgs.openssh}/bin/ssh -NT -i ${config.sops.secrets."minecraft/wow-tunnel-key".path} -o IPQoS=none -o Ciphers=aes256-ctr -o MACs=hmac-sha2-256-etm@openssh.com -o KexAlgorithms=curve25519-sha256 -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ssh/ssh_known_hosts -o ExitOnForwardFailure=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o ConnectTimeout=10 -R 127.0.0.1:25577:127.0.0.1:25567 mc-wow-relay@${relay.address}";
|
||||
Restart = "always";
|
||||
RestartSec = 10;
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
};
|
||||
};
|
||||
in {
|
||||
users.groups.mc-wow-tunnel = { };
|
||||
users.users.mc-wow-tunnel = {
|
||||
isSystemUser = true;
|
||||
group = "mc-wow-tunnel";
|
||||
};
|
||||
sops.secrets."minecraft/wow-tunnel-key" = {
|
||||
sopsFile = ../../../../sus/neuro-minecraft.yaml;
|
||||
owner = "mc-wow-tunnel";
|
||||
group = "mc-wow-tunnel";
|
||||
mode = "0400";
|
||||
};
|
||||
programs.ssh.knownHosts.minecraft-wow-relay = {
|
||||
hostNames = [ "128.140.75.58" ];
|
||||
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAFpr4DPSaJt0xeuGIfcZBJD3LsJHTdIRIs2Tt9HF+CT";
|
||||
};
|
||||
programs.ssh.knownHosts.minecraft-wow-relay-bfs = {
|
||||
hostNames = [ "91.198.166.181" ];
|
||||
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICcCn57nlWY5QyEz17kxuAbIX9PkjPwtlGzdJyhy+SQQ";
|
||||
};
|
||||
systemd.services.minecraft-wow-tunnel = mkTunnel {
|
||||
name = "hectic-lab";
|
||||
address = "128.140.75.58";
|
||||
};
|
||||
systemd.services.minecraft-wow-tunnel-bfs = mkTunnel {
|
||||
name = "bfs.band";
|
||||
address = "91.198.166.181";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{ pkgs, ... }:
|
||||
|
||||
{
|
||||
services.minecraft-servers.servers.vanilla = {
|
||||
enable = true;
|
||||
jvmOpts = "-Xmx6G -Xms2G";
|
||||
package = pkgs.minecraftServers.vanilla-1_21_11;
|
||||
|
||||
serverProperties = {
|
||||
server-port = 25565;
|
||||
difficulty = "hard";
|
||||
online-mode = true;
|
||||
view-distance = 32;
|
||||
level-seed = "8306359138650378643";
|
||||
pause-when-empty-seconds = 0;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) mkOption types;
|
||||
|
||||
cfg = config.services.minecraft-world-imports;
|
||||
enabledImports = lib.filterAttrs (_: import: import.enable) cfg;
|
||||
dataDir = config.services.minecraft-servers.dataDir;
|
||||
minecraftServers = config.services.minecraft-servers.servers;
|
||||
targetPairs = lib.mapAttrsToList (_: import:
|
||||
"${import.serverName}:${import.worldName}") enabledImports;
|
||||
|
||||
importerUser = name: let
|
||||
descriptiveName = "minecraft-map-import-${name}";
|
||||
in
|
||||
if builtins.stringLength descriptiveName <= 31
|
||||
then descriptiveName
|
||||
else "mc-import-${builtins.substring 0 12 (builtins.hashString "sha256" name)}";
|
||||
|
||||
stateDirectory = cacheDir:
|
||||
if lib.hasPrefix "/var/lib/" cacheDir
|
||||
then lib.removePrefix "/var/lib/" cacheDir
|
||||
else null;
|
||||
|
||||
stateDirectoryCompatible = cacheDir: let
|
||||
relative = stateDirectory cacheDir;
|
||||
components = lib.splitString "/" (if relative == null then "" else relative);
|
||||
in
|
||||
relative != null
|
||||
&& relative != ""
|
||||
&& lib.all (component: component != "" && component != "." && component != "..") components;
|
||||
|
||||
escapeSftp = value:
|
||||
"\"${lib.replaceStrings ["\\" "\""] ["\\\\" "\\\""] value}\"";
|
||||
|
||||
escapeTmpfiles = value:
|
||||
lib.replaceStrings ["%" " " "\t"] ["%%" "\\x20" "\\x09"] value;
|
||||
|
||||
importerServices = lib.mkMerge (lib.mapAttrsToList (name: import: let
|
||||
user = importerUser name;
|
||||
unitName = "minecraft-world-import-${name}";
|
||||
serverUnit = "minecraft-server-${import.serverName}.service";
|
||||
cacheStateDirectory = stateDirectory import.cacheDir;
|
||||
serverDir = "${dataDir}/${import.serverName}";
|
||||
worldDir = "${serverDir}/${import.worldName}";
|
||||
in {
|
||||
${unitName} = {
|
||||
description = "Import Minecraft world ${name}";
|
||||
before = [ serverUnit ];
|
||||
wants = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
path = [ pkgs.coreutils pkgs.findutils pkgs.openssh pkgs.unar ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = user;
|
||||
Group = user;
|
||||
RemainAfterExit = true;
|
||||
TimeoutStartSec = import.timeoutStartSec;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
NoNewPrivileges = true;
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
ReadWritePaths = [ import.cacheDir dataDir ];
|
||||
UMask = "0077";
|
||||
} // lib.optionalAttrs (stateDirectoryCompatible import.cacheDir) {
|
||||
StateDirectory = cacheStateDirectory;
|
||||
};
|
||||
script = ''
|
||||
set -eu
|
||||
umask 077
|
||||
|
||||
cache_dir=${lib.escapeShellArg import.cacheDir}
|
||||
server_dir=${lib.escapeShellArg serverDir}
|
||||
world_dir=${lib.escapeShellArg worldDir}
|
||||
archive_name=${lib.escapeShellArg import.archiveName}
|
||||
world_name=${lib.escapeShellArg import.worldName}
|
||||
archive="$cache_dir/$archive_name"
|
||||
temporary_archive="$cache_dir/.$archive_name.$$"
|
||||
extraction_dir="$cache_dir/.minecraft-world-import-${name}.$$"
|
||||
staged_world="$server_dir/.$world_name.import.$$"
|
||||
key=${lib.escapeShellArg import.sshKeyFile}
|
||||
|
||||
cleanup() {
|
||||
rm -f "$temporary_archive"
|
||||
rm -rf "$extraction_dir" "$staged_world"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$cache_dir" "$server_dir"
|
||||
chmod 0700 "$cache_dir"
|
||||
# The Minecraft module creates server_dir as minecraft:minecraft
|
||||
# with group write access; this importer must not chmod another user's directory.
|
||||
|
||||
if [ -d "$world_dir" ]; then
|
||||
if [ -f "$world_dir/level.dat" ]; then
|
||||
exit 0
|
||||
fi
|
||||
echo "Minecraft world directory exists but has no level.dat" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$archive" ]; then
|
||||
downloaded=false
|
||||
attempt=1
|
||||
while [ "$attempt" -le ${toString import.downloadRetries} ]; do
|
||||
if sftp \
|
||||
-o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=yes \
|
||||
-o UserKnownHostsFile=/etc/ssh/ssh_known_hosts \
|
||||
-i "$key" \
|
||||
-b - \
|
||||
${lib.escapeShellArg "${import.remoteUser}@${import.remoteHost}"} <<EOF
|
||||
get ${escapeSftp import.remotePath} "$temporary_archive"
|
||||
EOF
|
||||
then
|
||||
downloaded=true
|
||||
break
|
||||
fi
|
||||
rm -f "$temporary_archive"
|
||||
sleep ${toString import.retryDelaySeconds}
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
if [ "$downloaded" != true ]; then
|
||||
echo "Unable to download Minecraft world ${name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
mv "$temporary_archive" "$archive"
|
||||
fi
|
||||
|
||||
if ! printf '%s %s\n' \
|
||||
${lib.escapeShellArg import.archiveSha256} \
|
||||
"$archive" | sha256sum -c -; then
|
||||
rm -f "$archive"
|
||||
echo "Cached Minecraft world ${name} checksum mismatch" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$extraction_dir"
|
||||
unar -quiet -output-directory "$extraction_dir" "$archive"
|
||||
|
||||
find "$extraction_dir" \
|
||||
\( -type l -o -type b -o -type c -o -type p -o -type s \) \
|
||||
-delete
|
||||
|
||||
world_level_dat=$(find "$extraction_dir" -type f -name level.dat -print -quit)
|
||||
if [ -z "$world_level_dat" ]; then
|
||||
echo "Minecraft world archive contains no level.dat" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mv "$(dirname "$world_level_dat")" "$staged_world"
|
||||
chgrp -R minecraft "$staged_world"
|
||||
chmod -R u+rwX,g+rwX,o-rwx "$staged_world"
|
||||
mv "$staged_world" "$world_dir"
|
||||
'';
|
||||
};
|
||||
|
||||
"minecraft-server-${import.serverName}" = {
|
||||
requires = [ "${unitName}.service" ];
|
||||
after = [ "${unitName}.service" ];
|
||||
};
|
||||
}) enabledImports);
|
||||
in {
|
||||
options.services.minecraft-world-imports = mkOption {
|
||||
default = { };
|
||||
type = types.attrsOf (types.submodule ({ name, ... }: {
|
||||
options = {
|
||||
enable = lib.mkEnableOption "Minecraft world import ${name}";
|
||||
|
||||
serverName = mkOption {
|
||||
type = types.str;
|
||||
description = "minecraft-servers server receiving imported world";
|
||||
};
|
||||
|
||||
remoteHost = mkOption {
|
||||
type = types.str;
|
||||
description = "SSH host serving world archive";
|
||||
};
|
||||
|
||||
remoteUser = mkOption {
|
||||
type = types.str;
|
||||
description = "SSH user used to download world archive";
|
||||
};
|
||||
|
||||
remotePath = mkOption {
|
||||
type = types.str;
|
||||
description = "Remote path to world archive";
|
||||
};
|
||||
|
||||
archiveName = mkOption {
|
||||
type = types.str;
|
||||
description = "Archive file name inside cache directory";
|
||||
};
|
||||
|
||||
cacheDir = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/minecraft-world-imports/${name}";
|
||||
description = "Persistent archive cache directory";
|
||||
};
|
||||
|
||||
archiveSha256 = mkOption {
|
||||
type = types.strMatching "[0-9a-fA-F]{64}";
|
||||
description = "Expected SHA-256 digest of world archive";
|
||||
};
|
||||
|
||||
sshKeyFile = mkOption {
|
||||
type = types.str;
|
||||
description = "Runtime path to private SSH key";
|
||||
};
|
||||
|
||||
worldName = mkOption {
|
||||
type = types.str;
|
||||
default = "world";
|
||||
description = "World directory name beneath server directory";
|
||||
};
|
||||
|
||||
hostPublicKey = mkOption {
|
||||
type = types.str;
|
||||
description = "Pinned SSH host public key";
|
||||
};
|
||||
|
||||
downloadRetries = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 3;
|
||||
description = "Maximum SFTP download attempts";
|
||||
};
|
||||
|
||||
retryDelaySeconds = mkOption {
|
||||
type = types.ints.unsigned;
|
||||
default = 10;
|
||||
description = "Delay between SFTP download attempts";
|
||||
};
|
||||
|
||||
timeoutStartSec = mkOption {
|
||||
type = types.str;
|
||||
default = "30min";
|
||||
description = "Importer service start timeout";
|
||||
};
|
||||
};
|
||||
}));
|
||||
description = "Minecraft worlds imported before selected servers start";
|
||||
};
|
||||
|
||||
config = lib.mkIf (enabledImports != { }) {
|
||||
assertions = lib.flatten (lib.mapAttrsToList (name: import: [
|
||||
{
|
||||
assertion = builtins.match "[A-Za-z0-9_-]+" name != null;
|
||||
message = "services.minecraft-world-imports.${name}: name must contain only letters, digits, underscores, or hyphens";
|
||||
}
|
||||
{
|
||||
assertion = builtins.stringLength name <= 24;
|
||||
message = "services.minecraft-world-imports.${name}: name must be at most 24 characters";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "/.*" import.cacheDir != null;
|
||||
message = "services.minecraft-world-imports.${name}.cacheDir must be absolute";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "[A-Za-z0-9_-]+" import.serverName != null;
|
||||
message = "services.minecraft-world-imports.${name}.serverName must contain only letters, digits, underscores, or hyphens";
|
||||
}
|
||||
{
|
||||
assertion = !lib.hasInfix "\n" import.remotePath && !lib.hasInfix "\r" import.remotePath;
|
||||
message = "services.minecraft-world-imports.${name}.remotePath must not contain newlines";
|
||||
}
|
||||
{
|
||||
assertion = builtins.hasAttr import.serverName minecraftServers
|
||||
&& (builtins.getAttr import.serverName minecraftServers).enable;
|
||||
message = "services.minecraft-world-imports.${name}.serverName must name an enabled Minecraft server";
|
||||
}
|
||||
{
|
||||
assertion = lib.length (lib.unique targetPairs) == lib.length targetPairs;
|
||||
message = "services.minecraft-world-imports: each server/world target must be unique";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "[^/]+" import.archiveName != null;
|
||||
message = "services.minecraft-world-imports.${name}.archiveName must be a file name";
|
||||
}
|
||||
{
|
||||
assertion = builtins.match "[^/]+" import.worldName != null;
|
||||
message = "services.minecraft-world-imports.${name}.worldName must be a directory name";
|
||||
}
|
||||
]) enabledImports);
|
||||
|
||||
users.groups = lib.mapAttrs' (name: _: lib.nameValuePair (importerUser name) { }) enabledImports;
|
||||
users.users = lib.mapAttrs' (name: _: let
|
||||
user = importerUser name;
|
||||
in lib.nameValuePair user {
|
||||
description = "Minecraft world importer ${name}";
|
||||
isSystemUser = true;
|
||||
group = user;
|
||||
extraGroups = [ "minecraft" ];
|
||||
}) enabledImports;
|
||||
|
||||
programs.ssh.knownHosts = lib.mapAttrs' (name: import:
|
||||
lib.nameValuePair "minecraft-world-import-${name}" {
|
||||
hostNames = [ import.remoteHost ];
|
||||
publicKey = import.hostPublicKey;
|
||||
}) enabledImports;
|
||||
|
||||
systemd.tmpfiles.rules = lib.flatten (lib.mapAttrsToList (name: import:
|
||||
lib.optional (!stateDirectoryCompatible import.cacheDir)
|
||||
"d ${escapeTmpfiles import.cacheDir} 0700 ${importerUser name} ${importerUser name} -") enabledImports);
|
||||
|
||||
systemd.services = importerServices;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{ config, ... }:
|
||||
|
||||
{
|
||||
sops.secrets."minecraft/storage-box-pack-key" = {
|
||||
sopsFile = ../../../../sus/neuro-minecraft.yaml;
|
||||
owner = "mc-pack-worldOfSosal";
|
||||
group = "mc-pack-worldOfSosal";
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
services.minecraft-modpack-imports.worldOfSosal = {
|
||||
enable = true;
|
||||
serverName = "wowMineMap";
|
||||
remoteHost = "u664722.your-storagebox.de";
|
||||
remoteUser = "u664722";
|
||||
remotePath = "minecraft/pack/WorldOfSosal-v3.mrpack";
|
||||
archiveName = "WorldOfSosal.mrpack";
|
||||
cacheDir = "/var/lib/minecraft-modpacks/worldOfSosal";
|
||||
archiveSha256 = "f97cf251b14f40590e97e7b39e8a8ec43dacfce6da1b02357d15e0eee10d3ade";
|
||||
expectedDependencies = {
|
||||
minecraft = "1.21.1";
|
||||
neoforge = "21.1.250";
|
||||
};
|
||||
sshKeyFile = config.sops.secrets."minecraft/storage-box-pack-key".path;
|
||||
hostPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
|
||||
};
|
||||
|
||||
# Import the map before writing modpack configuration into the same server.
|
||||
systemd.services.minecraft-modpack-import-worldOfSosal = {
|
||||
after = [ "minecraft-world-import-wowMineMap.service" ];
|
||||
requires = [ "minecraft-world-import-wowMineMap.service" ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
{ config, pkgs, ... }:
|
||||
|
||||
{
|
||||
sops.secrets."minecraft/storage-box-key" = {
|
||||
sopsFile = ../../../../sus/neuro-minecraft.yaml;
|
||||
owner = "minecraft-map-import-wowMineMap";
|
||||
group = "minecraft-map-import-wowMineMap";
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
services.minecraft-world-imports.wowMineMap = {
|
||||
enable = true;
|
||||
serverName = "wowMineMap";
|
||||
remoteHost = "u664722.your-storagebox.de";
|
||||
remoteUser = "u664722";
|
||||
remotePath = "minecraft/map/wow mine map.rar";
|
||||
archiveName = "wow mine map.rar";
|
||||
cacheDir = "/var/lib/minecraft-maps";
|
||||
archiveSha256 = "bc80084de10a06b0fc2cb1651c61936b9e2fd2288f3f0fe44c964d83a393aa30";
|
||||
sshKeyFile = config.sops.secrets."minecraft/storage-box-key".path;
|
||||
worldName = "world";
|
||||
hostPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
|
||||
};
|
||||
|
||||
sops.secrets."minecraft/rcon-password" = {
|
||||
sopsFile = ../../../../sus/neuro-minecraft.yaml;
|
||||
owner = "minecraft";
|
||||
group = "minecraft";
|
||||
mode = "0400";
|
||||
restartUnits = [ "minecraft-server-wowMineMap.service" ];
|
||||
};
|
||||
|
||||
# The module's automatic firewall would also expose RCON.
|
||||
networking.firewall.allowedTCPPorts = [ 25567 ];
|
||||
services.minecraft-servers.servers.wowMineMap = {
|
||||
openFirewall = false;
|
||||
extraStartPre = ''
|
||||
chmod 600 server.properties
|
||||
{
|
||||
printf '\nrcon.password='
|
||||
cat ${config.sops.secrets."minecraft/rcon-password".path}
|
||||
printf '\n'
|
||||
} >> server.properties
|
||||
'';
|
||||
enable = true;
|
||||
jvmOpts = "-Xmx24G -Xms2G";
|
||||
# WorldOfSosal client and server use the same pinned NeoForge.
|
||||
package = pkgs.minecraftServers.neoforge-1_21_1.override (
|
||||
builtins.fromJSON (builtins.readFile ./neoforge-21.1.250.json)
|
||||
);
|
||||
|
||||
serverProperties = {
|
||||
server-port = 25567;
|
||||
difficulty = "hard";
|
||||
online-mode = false;
|
||||
enable-rcon = true;
|
||||
"rcon.port" = 25575;
|
||||
view-distance = 12;
|
||||
simulation-distance = 8;
|
||||
motd = "WorldOfSosal — World of Warcraft";
|
||||
level-name = "world";
|
||||
pause-when-empty-seconds = 0;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -24,11 +24,11 @@
|
||||
|
||||
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
|
||||
pname = "ollama";
|
||||
version = "0.22.1";
|
||||
version = "0.24.0";
|
||||
|
||||
src = pkgs.fetchurl {
|
||||
url = "https://github.com/ollama/ollama/releases/download/v0.22.1/ollama-linux-amd64.tar.zst";
|
||||
hash = "sha256-4nwP6PYKgkFi+Bzge0v9p2fc5PNX12LhSbPQ3gq62fs=";
|
||||
url = "https://github.com/ollama/ollama/releases/download/v0.24.0/ollama-linux-amd64.tar.zst";
|
||||
hash = "sha256-FcX41mugbg07RxnfiGhhLb1m4U6CdgkpuzVS4WV83Ns=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
@@ -70,7 +70,7 @@ in {
|
||||
imports = [
|
||||
self.nixosModules.hectic
|
||||
inputs.sops-nix.nixosModules.sops
|
||||
./minecraft.nix
|
||||
./minecraft
|
||||
./hardware.nix
|
||||
];
|
||||
|
||||
@@ -139,7 +139,7 @@ in {
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
enable = false;
|
||||
virtualHosts."bfs.band" = let
|
||||
site = pkgs.runCommand "bfs-band-site" {} ''
|
||||
mkdir -p $out
|
||||
@@ -224,6 +224,70 @@ in {
|
||||
archetype.dev.enable = true;
|
||||
};
|
||||
|
||||
hectic.services."project-zomboid" = {
|
||||
enable = true;
|
||||
memory = "8g";
|
||||
serverName = "servertest";
|
||||
serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
|
||||
serverProperties = {
|
||||
Map = "Muldraugh, KY";
|
||||
DoLuaChecksum = false;
|
||||
Public = true;
|
||||
AntiCheatSafety = 4;
|
||||
AntiCheatMovement = 4;
|
||||
AntiCheatSpeed = 4;
|
||||
AntiCheatHit = 4;
|
||||
AntiCheatPacket = 4;
|
||||
AntiCheatPacketException = 4;
|
||||
AntiCheatPermission = 4;
|
||||
AntiCheatXP = 4;
|
||||
AntiCheatFire = 4;
|
||||
AntiCheatSafeHouse = 4;
|
||||
AntiCheatRecipe = 4;
|
||||
AntiCheatPlayer = 4;
|
||||
AntiCheatChecksum = 4;
|
||||
AntiCheatItem = 4;
|
||||
AntiCheatNoClip = 4;
|
||||
AntiCheatServerCustomization = 4;
|
||||
};
|
||||
workshopItems = [
|
||||
"3676456221" # Lua Digital Watch Framework
|
||||
"3600401184" # Realistic Temperature Mod
|
||||
];
|
||||
mods = [
|
||||
"\\LuaDigitalWatchUI"
|
||||
"\\RC_RealisticColdMod"
|
||||
];
|
||||
sandboxProperties = {
|
||||
Zombies = 6;
|
||||
ZombieConfig = {
|
||||
PopulationMultiplier = 0.0;
|
||||
PopulationStartMultiplier = 0.0;
|
||||
PopulationPeakMultiplier = 0.0;
|
||||
RespawnHours = 0.0;
|
||||
RespawnUnseenHours = 0.0;
|
||||
RespawnMultiplier = 0.0;
|
||||
RedistributeHours = 0.0;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.project-zomboid.preStart = lib.mkBefore ''
|
||||
password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"}
|
||||
properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"}
|
||||
|
||||
if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then
|
||||
umask 077
|
||||
${pkgs.openssl}/bin/openssl rand -hex 24 > "$password_file"
|
||||
fi
|
||||
${pkgs.coreutils}/bin/chmod 0600 "$password_file"
|
||||
|
||||
properties_file_tmp="$( ${pkgs.coreutils}/bin/mktemp "$(dirname "$properties_file")/.server-password.ini.XXXXXX")"
|
||||
${pkgs.coreutils}/bin/printf 'Password=%s\n' "$(<"$password_file")" > "$properties_file_tmp"
|
||||
${pkgs.coreutils}/bin/chmod 0600 "$properties_file_tmp"
|
||||
${pkgs.coreutils}/bin/mv "$properties_file_tmp" "$properties_file"
|
||||
'';
|
||||
|
||||
sops = {
|
||||
gnupg.sshKeyPaths = [ ];
|
||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||
|
||||
+2
-3
@@ -25,11 +25,10 @@
|
||||
rev = "6ff3b71e3705e0d4081a51c21ca0379e869ba5fb";
|
||||
hash = "sha256-wC/2rAsSDO83UITaFhtaf3do3aaOAko4gnKUOzwURc8=";
|
||||
};
|
||||
cargo = self.lib.cargoToml src;
|
||||
in
|
||||
buildPgrxExtension pkgs {
|
||||
pname = cargo.package.name;
|
||||
version = cargo.package.version;
|
||||
pname = "pg_smtp_client";
|
||||
version = "0.2.0";
|
||||
|
||||
inherit src postgresql;
|
||||
|
||||
|
||||
@@ -3,13 +3,9 @@
|
||||
# Owns: TTL sweep, orphan-VM sweep, deferred-job retry, stale-runner dereg,
|
||||
# startup convergence. Runs forever under systemd; webhook service is separate.
|
||||
|
||||
gcr_ttl_grace_sec() {
|
||||
printf '%s' "$((10 * 60))"
|
||||
}
|
||||
|
||||
gcr_record_age_sec() {
|
||||
created_at="$(gcr_record_field "$1" created_at)"
|
||||
now="$(date -u '+%s')"
|
||||
now="$(gcr_now_epoch)"
|
||||
case "$created_at" in
|
||||
''|*[!0-9]*) printf '%s' 999999 ;;
|
||||
*) printf '%s' "$((now - created_at))" ;;
|
||||
@@ -20,33 +16,100 @@ gcr_sweep_ttl() {
|
||||
for f in $(gcr_active_records); do
|
||||
rec="$(cat "$f")"
|
||||
status="$(gcr_record_field "$rec" status)"
|
||||
[ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue
|
||||
|
||||
job_id="$(gcr_record_field "$rec" job_id)"
|
||||
attempt="$(gcr_record_field "$rec" run_attempt)"
|
||||
|
||||
if [ "$status" = "idle_vm" ]; then
|
||||
gcr_lock_acquire idle-pool || continue
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ -z "$rec" ] || [ "$(gcr_record_field "$rec" status)" != "idle_vm" ]; then
|
||||
gcr_lock_release idle-pool
|
||||
continue
|
||||
fi
|
||||
if gcr_idle_record_unexpired "$rec"; then
|
||||
gcr_lock_release idle-pool
|
||||
continue
|
||||
fi
|
||||
|
||||
vm_id="$(gcr_record_field "$rec" vm_id)"
|
||||
if gcr_record_vm_owned_elsewhere "$vm_id" "$job_id" "$attempt"; then
|
||||
gcr_log warn --ns=sweep "removing superseded idle record job=$job_id vm=$vm_id"
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release idle-pool
|
||||
continue
|
||||
fi
|
||||
gcr_log info --ns=sweep "idle slot expired job=$job_id vm=$vm_id"
|
||||
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
|
||||
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-expired false; then
|
||||
gcr_event "vm-destroyed" "$job_id" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
|
||||
else
|
||||
gcr_event "vm-cleanup-pending" "$job_id" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
|
||||
fi
|
||||
else
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
fi
|
||||
gcr_lock_release idle-pool
|
||||
continue
|
||||
fi
|
||||
|
||||
[ "$status" = "vm_active" ] || [ "$status" = "pending_vm" ] || continue
|
||||
ttl_min="$(gcr_record_field "$rec" ttl_min)"
|
||||
case "$ttl_min" in ''|*[!0-9]*) continue ;; esac
|
||||
|
||||
max_sec="$((ttl_min * 60 + $(gcr_ttl_grace_sec)))"
|
||||
max_sec="$((ttl_min * 60))"
|
||||
age="$(gcr_record_age_sec "$rec")"
|
||||
if [ "$age" -gt "$max_sec" ]; then
|
||||
if [ "$age" -ge "$max_sec" ]; then
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || continue
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
case "$(gcr_record_field "$rec" status)" in
|
||||
pending_vm|vm_active) ;;
|
||||
*) gcr_lock_release "$key"; continue ;;
|
||||
esac
|
||||
ttl_min="$(gcr_record_field "$rec" ttl_min)"
|
||||
case "$ttl_min" in
|
||||
''|*[!0-9]*) gcr_lock_release "$key"; continue ;;
|
||||
esac
|
||||
max_sec="$((ttl_min * 60))"
|
||||
age="$(gcr_record_age_sec "$rec")"
|
||||
if [ "$age" -lt "$max_sec" ]; then
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
|
||||
vm_id="$(gcr_record_field "$rec" vm_id)"
|
||||
gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s"
|
||||
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
|
||||
ip="$(gcr_vm_public_ip "$vm_id" || true)"
|
||||
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true
|
||||
gcr_vm_destroy "$vm_id" || true
|
||||
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
|
||||
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" ttl false; then
|
||||
gcr_event "vm-destroyed" "$job_id" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
|
||||
else
|
||||
gcr_event "vm-cleanup-pending" "$job_id" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
|
||||
fi
|
||||
else
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
fi
|
||||
gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}"
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_release "$key"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
gcr_sweep_orphan_vms() {
|
||||
vms_json="$(gcr_vm_list_managed)" || return 0
|
||||
gcr_lock_acquire admission || return 0
|
||||
vms_json="$(gcr_vm_list_managed)" || {
|
||||
gcr_lock_release admission
|
||||
return 0
|
||||
}
|
||||
if ! gcr_lock_acquire idle-pool; then
|
||||
gcr_lock_release admission
|
||||
return 0
|
||||
fi
|
||||
count="$(printf '%s' "$vms_json" | jq 'length')"
|
||||
i=0
|
||||
while [ "$i" -lt "$count" ]; do
|
||||
@@ -55,74 +118,211 @@ gcr_sweep_orphan_vms() {
|
||||
jid="$(printf '%s' "$vm" | jq -r '.labels["gcr.job-id"] // ""')"
|
||||
att="$(printf '%s' "$vm" | jq -r '.labels["gcr.run-attempt"] // ""')"
|
||||
|
||||
known=""
|
||||
if [ -n "$jid" ] && [ -n "$att" ]; then
|
||||
rec="$(gcr_record_get "$jid" "$att")"
|
||||
[ -n "$rec" ] && known=1
|
||||
fi
|
||||
|
||||
if [ -z "$known" ]; then
|
||||
if ! gcr_record_exists_for_vm_id "$vm_id"; then
|
||||
gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy"
|
||||
gcr_vm_destroy "$vm_id" || true
|
||||
gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}"
|
||||
cleanup_job="${jid:-orphan-$vm_id}"
|
||||
cleanup_attempt="${att:-0}"
|
||||
cleanup_rec="$(jq -n --arg j "$cleanup_job" --arg a "$cleanup_attempt" \
|
||||
--arg v "$vm_id" --arg vn "$(printf '%s' "$vm" | jq -r '.name // ""')" \
|
||||
'{job_id:$j, run_attempt:$a, repo:"", label:"", created_at:"0",
|
||||
ttl_min:0, vm_id:($v|tonumber), vm_name:$vn,
|
||||
bootstrapped:false, status:"cleanup_pending"}')"
|
||||
if gcr_vm_cleanup_start "$cleanup_job" "$cleanup_attempt" \
|
||||
"$cleanup_rec" orphan false; then
|
||||
gcr_event "orphan-vm-destroyed" "$cleanup_job" "{\"vm_id\":$vm_id}"
|
||||
else
|
||||
gcr_event "vm-cleanup-pending" "$cleanup_job" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"orphan\"}"
|
||||
fi
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
gcr_lock_release idle-pool
|
||||
gcr_lock_release admission
|
||||
}
|
||||
|
||||
gcr_alloc_deferred() {
|
||||
job_id="$1"; attempt="$2"
|
||||
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || return 0
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
[ -n "$rec" ] || return 0
|
||||
[ "$(gcr_record_field "$rec" status)" = "deferred" ] || return 0
|
||||
if [ -z "$rec" ] || [ "$(gcr_record_field "$rec" status)" != "deferred" ]; then
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
fi
|
||||
|
||||
repo="$(gcr_record_field "$rec" repo)"
|
||||
label="$(gcr_record_field "$rec" label)"
|
||||
|
||||
state="$(gcr_gitea_job_state "$repo" "$job_id")" || return 0
|
||||
state="$(gcr_gitea_job_state "$repo" "$job_id")" || {
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
}
|
||||
case "$state" in
|
||||
completed:*)
|
||||
gcr_log info --ns=alloc "deferred job=$job_id already terminal ($state), dropping record"
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
profile="$(gcr_label_profile "$label")" || return 0
|
||||
profile="$(gcr_label_profile "$label")" || {
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
}
|
||||
set -- $profile
|
||||
server_type="$1"; ttl_min="$2"; rate="$3"
|
||||
server_type="$1"; ttl_min="$2"
|
||||
|
||||
gcr_lock_acquire admission || {
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
}
|
||||
active="$(gcr_count_active)"
|
||||
repo_active="$(gcr_count_active_repo "$repo")"
|
||||
[ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] && return 0
|
||||
[ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ] && return 0
|
||||
if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \
|
||||
|| [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
fi
|
||||
|
||||
gcr_budget_add "$rate" "$ttl_min" || return 0
|
||||
reg_token="$(gcr_gitea_registration_token "$repo")" || return 0
|
||||
claim_status=0
|
||||
gcr_claim_idle "$job_id" "$attempt" "$repo" "$label" || claim_status="$?"
|
||||
if [ "$claim_status" -eq 0 ]; then
|
||||
reused="$(gcr_record_get "$job_id" "$attempt")"
|
||||
vm_id="$(gcr_record_field "$reused" vm_id)"
|
||||
if gcr_vm_runner_service "$vm_id" start \
|
||||
&& gcr_vm_runner_service "$vm_id" health \
|
||||
&& gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" false; then
|
||||
reused="$(gcr_record_get "$job_id" "$attempt")"
|
||||
reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')"
|
||||
gcr_record_put "$job_id" "$attempt" "$reused"
|
||||
else
|
||||
gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" true || true
|
||||
gcr_event "vm-reuse-start-failed" "$job_id" "{\"vm_id\":$vm_id,\"via\":\"deferred-retry\"}"
|
||||
fi
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "vm-reused" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"via\":\"deferred-retry\"}"
|
||||
gcr_log info --ns=alloc "deferred job=$job_id reused vm=$vm_id"
|
||||
return 0
|
||||
fi
|
||||
if [ "$claim_status" -eq 2 ]; then
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
fi
|
||||
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || return 0
|
||||
|
||||
vm_name="gcr-${job_id}-${attempt}"
|
||||
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
|
||||
"$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || {
|
||||
reg_token="$(gcr_gitea_registration_token "$repo")" || {
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
}
|
||||
|
||||
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
|
||||
--arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \
|
||||
--arg vn "$vm_name" --arg ttl "$ttl_min" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
|
||||
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
vm_name="gcr-${job_id}-${attempt}"
|
||||
created_at="$(gcr_now_epoch)"
|
||||
create_status=0
|
||||
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
|
||||
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
|
||||
if [ "$create_status" -ne 0 ] || [ -z "$created" ]; then
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
fi
|
||||
set -- $created
|
||||
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
|
||||
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
|
||||
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
|
||||
"$actual_server_type" "$actual_rate"; then
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
fi
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min,\"via\":\"deferred-retry\"}"
|
||||
gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id"
|
||||
}
|
||||
|
||||
gcr_sweep_cleanup_pending() {
|
||||
for f in $(gcr_active_records); do
|
||||
rec="$(cat "$f")"
|
||||
[ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ] || continue
|
||||
job_id="$(gcr_record_field "$rec" job_id)"
|
||||
attempt="$(gcr_record_field "$rec" run_attempt)"
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || continue
|
||||
if ! gcr_lock_acquire admission; then
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ]; then
|
||||
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
|
||||
fi
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
done
|
||||
}
|
||||
|
||||
gcr_sweep_create_ambiguous() {
|
||||
for f in $(gcr_active_records); do
|
||||
rec="$(cat "$f")"
|
||||
[ "$(gcr_record_field "$rec" status)" = "create_ambiguous" ] || continue
|
||||
job_id="$(gcr_record_field "$rec" job_id)"
|
||||
attempt="$(gcr_record_field "$rec" run_attempt)"
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || continue
|
||||
if ! gcr_lock_acquire admission; then
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ "$(gcr_record_field "$rec" status)" != "create_ambiguous" ]; then
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
find_status=0
|
||||
found_vm_id="$(gcr_vm_find_created \
|
||||
"$(gcr_record_field "$rec" vm_name)" "$job_id" "$attempt" \
|
||||
"$(gcr_record_field "$rec" label)" \
|
||||
"$(gcr_record_field "$rec" server_type)" \
|
||||
"$(gcr_record_field "$rec" candidate_location)" \
|
||||
"$(gcr_record_field "$rec" candidate_arch)")" || find_status="$?"
|
||||
case "$find_status" in
|
||||
0)
|
||||
rec="$(printf '%s' "$rec" | jq -c --arg vm "$found_vm_id" \
|
||||
'.vm_id = ($vm | tonumber)
|
||||
| .status = "pending_vm"
|
||||
| .bootstrapped = false
|
||||
| del(.create_http, .create_curl_status,
|
||||
.candidate_location, .candidate_arch)')"
|
||||
if gcr_record_put "$job_id" "$attempt" "$rec"; then
|
||||
gcr_event "vm-create-recovered" "$job_id" \
|
||||
"{\"vm_id\":$found_vm_id,\"label\":\"$(gcr_record_field "$rec" label)\"}"
|
||||
fi
|
||||
;;
|
||||
1)
|
||||
rec="$(printf '%s' "$rec" | jq -c \
|
||||
'.status = "cleanup_pending"
|
||||
| .cleanup_reason = "ambiguous-create-absent"
|
||||
| .cleanup_refund_budget = true
|
||||
| .cleanup_vm_destroyed = true
|
||||
| .cleanup_budget_released = false')"
|
||||
if gcr_record_put "$job_id" "$attempt" "$rec"; then
|
||||
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
|
||||
fi
|
||||
;;
|
||||
2) ;;
|
||||
esac
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
done
|
||||
}
|
||||
|
||||
gcr_retry_deferred() {
|
||||
for f in $(gcr_active_records); do
|
||||
rec="$(cat "$f")"
|
||||
@@ -134,38 +334,56 @@ gcr_retry_deferred() {
|
||||
}
|
||||
|
||||
gcr_sweep_stale_runners() {
|
||||
runners="$(gcr_gitea_list_runners)" || return 0
|
||||
# Here-doc instead of pipe: dash runs pipe tails in a subshell, which
|
||||
# would strand gcr_event/audit writes from the caller's perspective.
|
||||
while read -r rid rname; do
|
||||
[ -n "${rid:-}" ] || continue
|
||||
case "$rname" in
|
||||
gcr-*) ;;
|
||||
gcr_lock_acquire idle-pool || return 0
|
||||
oldIFS="$IFS"
|
||||
IFS=,
|
||||
for allowed_repo in ${GCR_ALLOWED_REPOS:-}; do
|
||||
IFS="$oldIFS"
|
||||
case "$allowed_repo" in
|
||||
*/\*)
|
||||
owner="${allowed_repo%/*}"
|
||||
repos="$(gcr_gitea_list_org_repos "$owner")" || {
|
||||
IFS=,
|
||||
continue
|
||||
}
|
||||
;;
|
||||
*/*) repos="$allowed_repo" ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
while read -r repo; do
|
||||
[ -n "${repo:-}" ] || continue
|
||||
gcr_repo_allowed "$repo" || continue
|
||||
runners="$(gcr_gitea_list_runners "$repo")" || continue
|
||||
# Here-doc instead of pipe: dash runs pipe tails in a subshell, which
|
||||
# would strand gcr_event/audit writes from the caller's perspective.
|
||||
while read -r rid rname; do
|
||||
[ -n "${rid:-}" ] || continue
|
||||
case "$rname" in
|
||||
gcr-*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
|
||||
# gcr-<job>-<attempt>: alive iff a matching active/pending record exists.
|
||||
rest="${rname#gcr-}"
|
||||
jid="${rest%-*}"
|
||||
att="${rest##*-}"
|
||||
rec=""
|
||||
case "$jid" in *[!0-9]*|"") rec="" ;;
|
||||
*) case "$att" in *[!0-9]*|"") rec="" ;;
|
||||
*) rec="$(gcr_record_get "$jid" "$att")" ;;
|
||||
esac ;;
|
||||
esac
|
||||
|
||||
if [ -z "$rec" ]; then
|
||||
gcr_log warn --ns=sweep "stale runner registration id=$rid name=$rname -> delete"
|
||||
if gcr_gitea_delete_runner "$rid"; then
|
||||
gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"runner_id\":$rid,\"name\":\"$rname\"}"
|
||||
else
|
||||
gcr_log error --ns=sweep "failed deleting runner id=$rid"
|
||||
# Runner name stays tied to original VM across later job assignments.
|
||||
rest="${rname#gcr-}"
|
||||
jid="${rest%-*}"
|
||||
if ! gcr_record_exists_for_vm_name "$rname"; then
|
||||
gcr_log warn --ns=sweep "stale repo=$repo registration id=$rid name=$rname -> delete"
|
||||
if gcr_gitea_delete_runner "$repo" "$rid"; then
|
||||
gcr_event "stale-runner-deleted" "${jid:-unknown}" "{\"repo\":\"$repo\",\"runner_id\":$rid,\"name\":\"$rname\"}"
|
||||
else
|
||||
gcr_log error --ns=sweep "failed deleting repo=$repo runner id=$rid"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done <<EOF
|
||||
done <<EOF
|
||||
$runners
|
||||
EOF
|
||||
done <<EOF
|
||||
$repos
|
||||
EOF
|
||||
IFS=,
|
||||
done
|
||||
IFS="$oldIFS"
|
||||
gcr_lock_release idle-pool
|
||||
}
|
||||
|
||||
# Runs SSH-push bootstrap for VMs that were created but not yet provisioned.
|
||||
@@ -178,12 +396,36 @@ gcr_bootstrap_pending() {
|
||||
|
||||
job_id="$(gcr_record_field "$rec" job_id)"
|
||||
attempt="$(gcr_record_field "$rec" run_attempt)"
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || continue
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ "$(gcr_record_field "$rec" status)" != "pending_vm" ] \
|
||||
|| [ "$(gcr_record_field "$rec" bootstrapped)" = "true" ]; then
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
repo="$(gcr_record_field "$rec" repo)"
|
||||
label="$(gcr_record_field "$rec" label)"
|
||||
vm_id="$(gcr_record_field "$rec" vm_id)"
|
||||
runner_name="$(gcr_record_field "$rec" vm_name)"
|
||||
|
||||
state="$(gcr_gitea_job_state "$repo" "$job_id")" || continue
|
||||
if [ "$(gcr_record_field "$rec" reused_vm)" = "true" ]; then
|
||||
if gcr_vm_runner_service "$vm_id" start \
|
||||
&& gcr_vm_runner_service "$vm_id" health \
|
||||
&& gcr_gitea_runner_disabled "$repo" "$runner_name" false; then
|
||||
rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | del(.reused_vm)')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
else
|
||||
gcr_gitea_runner_disabled "$repo" "$runner_name" true || true
|
||||
fi
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
|
||||
state="$(gcr_gitea_job_state "$repo" "$job_id")" || {
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
}
|
||||
case "$state" in
|
||||
completed:*)
|
||||
gcr_log info --ns=sweep "pending job=$job_id already terminal ($state), destroying vm=$vm_id"
|
||||
@@ -195,19 +437,35 @@ gcr_bootstrap_pending() {
|
||||
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
|
||||
;;
|
||||
esac
|
||||
gcr_vm_destroy "$vm_id" || true
|
||||
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
|
||||
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
|
||||
pending-job-completed false; then
|
||||
gcr_event "vm-destroyed" "$job_id" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
|
||||
else
|
||||
gcr_event "vm-cleanup-pending" "$job_id" \
|
||||
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\"}"
|
||||
fi
|
||||
else
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
fi
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
ip="$(gcr_vm_public_ip "$vm_id")"
|
||||
[ -n "$ip" ] || continue
|
||||
ip="$(gcr_vm_public_ip "$vm_id")" || {
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
}
|
||||
if [ -z "$ip" ]; then
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
|
||||
reg_token="$(gcr_gitea_registration_token "$repo")" || continue
|
||||
reg_token="$(gcr_gitea_registration_token "$repo")" || {
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
}
|
||||
ttl_min="$(gcr_record_field "$rec" ttl_min)"
|
||||
|
||||
gcr_log info --ns=alloc "bootstrapping vm=$vm_id ip=$ip job=$job_id"
|
||||
@@ -218,6 +476,7 @@ gcr_bootstrap_pending() {
|
||||
else
|
||||
gcr_log warn --ns=alloc "bootstrap failed vm=$vm_id (retry next tick)"
|
||||
fi
|
||||
gcr_lock_release "$key"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -238,26 +497,33 @@ gcr_reap_finished_jobs() {
|
||||
state="$(gcr_gitea_job_state "$repo" "$job_id")" || continue
|
||||
case "$state" in
|
||||
completed:*)
|
||||
gcr_log info --ns=sweep "job=$job_id terminal ($state), destroying vm=$vm_id"
|
||||
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
|
||||
case "$state" in
|
||||
completed:success|completed:cancelled|completed:skipped) ;;
|
||||
*)
|
||||
ip="$(gcr_vm_public_ip "$vm_id" || true)"
|
||||
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
|
||||
;;
|
||||
esac
|
||||
gcr_vm_destroy "$vm_id" || true
|
||||
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || continue
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ -z "$rec" ]; then
|
||||
gcr_lock_release "$key"
|
||||
continue
|
||||
fi
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
case "$(gcr_record_field "$rec" status)" in
|
||||
pending_vm|vm_active) ;;
|
||||
*) gcr_lock_release "$key"; continue ;;
|
||||
esac
|
||||
finish_status=0
|
||||
gcr_vm_finish_terminal "$job_id" "$attempt" "$rec" "$state" reconcile \
|
||||
|| finish_status="$?"
|
||||
gcr_lock_release "$key"
|
||||
case "$finish_status" in
|
||||
0|2) ;;
|
||||
*) return "$finish_status" ;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
gcr_tick() {
|
||||
gcr_sweep_create_ambiguous
|
||||
gcr_sweep_cleanup_pending
|
||||
gcr_sweep_ttl
|
||||
gcr_reap_finished_jobs
|
||||
gcr_sweep_orphan_vms
|
||||
|
||||
@@ -23,11 +23,12 @@ gcr_server_hourly_rate() {
|
||||
|
||||
gcr_label_ttl() {
|
||||
case "$1" in
|
||||
gross-x86) printf '180' ;;
|
||||
ubuntu-latest|gross-x86) printf '180' ;;
|
||||
gross-arm) printf '180' ;;
|
||||
gross-x86-perf) printf '180' ;;
|
||||
gross-mixed-econ) printf '180' ;;
|
||||
gross-nix-x86) printf '180' ;;
|
||||
nix) printf '480' ;;
|
||||
gross-nix-x86) printf '180' ;;
|
||||
gross-nix-arm) printf '180' ;;
|
||||
gross-nix-x86-perf) printf '480' ;;
|
||||
gross-nix-x86-highmem) printf '480' ;;
|
||||
@@ -40,6 +41,13 @@ gcr_label_ttl() {
|
||||
gcr_label_candidates() {
|
||||
label="$1"
|
||||
case "$label" in
|
||||
ubuntu-latest)
|
||||
printf '%s\n' \
|
||||
'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \
|
||||
'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \
|
||||
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
|
||||
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64'
|
||||
;;
|
||||
gross-x86)
|
||||
printf '%s\n' \
|
||||
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \
|
||||
@@ -64,6 +72,13 @@ gcr_label_candidates() {
|
||||
'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \
|
||||
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64'
|
||||
;;
|
||||
nix)
|
||||
printf '%s\n' \
|
||||
'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \
|
||||
'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \
|
||||
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
|
||||
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64'
|
||||
;;
|
||||
gross-nix-x86)
|
||||
printf '%s\n' \
|
||||
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \
|
||||
@@ -147,7 +162,7 @@ gcr_count_active() {
|
||||
for f in $(gcr_active_records); do
|
||||
status="$(gcr_record_field "$(cat "$f")" status)"
|
||||
case "$status" in
|
||||
pending_vm|vm_active) count=$((count + 1)) ;;
|
||||
pending_vm|vm_active|cleanup_pending|create_ambiguous) count=$((count + 1)) ;;
|
||||
esac
|
||||
done
|
||||
printf '%s' "$count"
|
||||
@@ -159,7 +174,7 @@ gcr_count_active_repo() {
|
||||
for f in $(gcr_active_records); do
|
||||
rec="$(cat "$f")"
|
||||
case "$(gcr_record_field "$rec" status)" in
|
||||
pending_vm|vm_active) ;;
|
||||
pending_vm|vm_active|cleanup_pending|create_ambiguous) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
[ "$(gcr_record_field "$rec" repo)" = "$repo" ] && count=$((count + 1))
|
||||
|
||||
@@ -20,12 +20,30 @@ gcr_gitea_admin_token() {
|
||||
tr -d '\n' < "$GITEA_ADMIN_TOKEN_FILE"
|
||||
}
|
||||
|
||||
# gcr_gitea_list_runners — prints "id name" lines for org hectic-lab.
|
||||
# gcr_gitea_list_runners REPO — prints "id name" lines for repo runners.
|
||||
gcr_gitea_list_runners() {
|
||||
repo="$1"
|
||||
token="$(gcr_gitea_admin_token)" || return 1
|
||||
owner="${repo%%/*}"
|
||||
name="${repo#*/}"
|
||||
curl -fsS -H "Authorization: token $token" \
|
||||
"$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners?per_page=50" \
|
||||
| jq -r '.entries[]? | "\(.id) \(.name)"'
|
||||
"$GCR_GITEA_URL/api/v1/repos/$owner/$name/actions/runners" \
|
||||
| jq -r '.runners[]? | "\(.id) \(.name)"'
|
||||
}
|
||||
|
||||
# gcr_gitea_list_org_repos OWNER — prints fully-qualified repository names.
|
||||
gcr_gitea_list_org_repos() {
|
||||
owner="$1"
|
||||
token="$(gcr_gitea_admin_token)" || return 1
|
||||
page=1
|
||||
while :; do
|
||||
repos="$(curl -fsS -H "Authorization: token $token" \
|
||||
"$GCR_GITEA_URL/api/v1/orgs/$owner/repos?page=$page&limit=50")" || return 1
|
||||
printf '%s' "$repos" | jq -r '.[]? | .full_name'
|
||||
count="$(printf '%s' "$repos" | jq 'length')" || return 1
|
||||
[ "$count" -lt 50 ] && return 0
|
||||
page=$((page + 1))
|
||||
done
|
||||
}
|
||||
|
||||
# gcr_gitea_job_state REPO JOB_ID — prints "<status>:<conclusion>".
|
||||
@@ -40,8 +58,35 @@ gcr_gitea_job_state() {
|
||||
}
|
||||
|
||||
gcr_gitea_delete_runner() {
|
||||
id="$1"
|
||||
repo="$1"; id="$2"
|
||||
token="$(gcr_gitea_admin_token)" || return 1
|
||||
owner="${repo%%/*}"
|
||||
name="${repo#*/}"
|
||||
curl -fsS -X DELETE -H "Authorization: token $token" \
|
||||
"$GCR_GITEA_URL/api/v1/orgs/hectic-lab/actions/runners/$id"
|
||||
"$GCR_GITEA_URL/api/v1/repos/$owner/$name/actions/runners/$id"
|
||||
}
|
||||
|
||||
gcr_gitea_set_runner_disabled() {
|
||||
repo="$1"; id="$2"; disabled="$3"
|
||||
case "$disabled" in true|false) ;; *) return 1 ;; esac
|
||||
token="$(gcr_gitea_admin_token)" || return 1
|
||||
owner="${repo%%/*}"
|
||||
name="${repo#*/}"
|
||||
curl -fsS -X PATCH -H "Authorization: token $token" \
|
||||
-H 'Content-Type: application/json' --data "{\"disabled\":$disabled}" \
|
||||
"$GCR_GITEA_URL/api/v1/repos/$owner/$name/actions/runners/$id" >/dev/null
|
||||
}
|
||||
|
||||
# gcr_gitea_runner_disabled REPO RUNNER_NAME true|false
|
||||
gcr_gitea_runner_disabled() {
|
||||
repo="$1"; runner_name="$2"; disabled="$3"
|
||||
runners="$(gcr_gitea_list_runners "$repo")" || return 1
|
||||
while read -r id name; do
|
||||
[ "$name" = "$runner_name" ] || continue
|
||||
gcr_gitea_set_runner_disabled "$repo" "$id" "$disabled"
|
||||
return "$?"
|
||||
done <<EOF
|
||||
$runners
|
||||
EOF
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ gcr_hcloud_req() {
|
||||
method="$1"; path="$2"; body="${3:-}"
|
||||
token="$(gcr_hcloud_token)" || return 1
|
||||
GCR_LAST_BODY="$(mktemp "${TMPDIR:-/tmp}/gcr-resp.XXXXXX")"
|
||||
curl_status=0
|
||||
if [ -n "$body" ]; then
|
||||
code="$(printf '%s' "$body" | curl -sS -X "$method" \
|
||||
-H "Authorization: Bearer $token" \
|
||||
@@ -43,13 +44,20 @@ gcr_hcloud_req() {
|
||||
--data-binary @- \
|
||||
-o "$GCR_LAST_BODY" \
|
||||
-w '%{http_code}' \
|
||||
"$GCR_API$path")"
|
||||
"$GCR_API$path")" || curl_status="$?"
|
||||
else
|
||||
code="$(curl -sS -X "$method" \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-o "$GCR_LAST_BODY" \
|
||||
-w '%{http_code}' \
|
||||
"$GCR_API$path")"
|
||||
"$GCR_API$path")" || curl_status="$?"
|
||||
fi
|
||||
case "$code" in ''|*[!0-9]*) code=000 ;; esac
|
||||
GCR_LAST_CURL_STATUS="$curl_status"
|
||||
GCR_LAST_HTTP="$code"
|
||||
if [ "$curl_status" -ne 0 ]; then
|
||||
gcr_log warn --ns=hcloud "transport failed path=$path curl=$curl_status http=$code"
|
||||
return 1
|
||||
fi
|
||||
case "$code" in 2??) return 0 ;; esac
|
||||
gcr_log warn --ns=hcloud "request failed path=$path http=$code body=$(head -c 200 "$GCR_LAST_BODY" | gcr_redact)"
|
||||
@@ -62,6 +70,64 @@ gcr_vm_list_managed() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Exact deterministic create identity. Exit 0 = one match (prints id),
|
||||
# 1 = confirmed absent, 2 = lookup failed or identity invariant violated.
|
||||
gcr_vm_find_created() {
|
||||
find_name="$1"; find_job="$2"; find_attempt="$3"; find_label="$4"
|
||||
find_type="$5"; find_location="$6"; find_arch="$7"
|
||||
if ! gcr_hcloud_req GET "/servers?name=$find_name"; then
|
||||
return 2
|
||||
fi
|
||||
find_matches="$(jq -c \
|
||||
--arg name "$find_name" --arg job "$find_job" --arg attempt "$find_attempt" \
|
||||
--arg label "$find_label" --arg type "$find_type" \
|
||||
--arg location "$find_location" --arg arch "$find_arch" \
|
||||
'[.servers[] | select(
|
||||
.name == $name
|
||||
and .labels["gitea-runner-controller"] == "managed"
|
||||
and .labels["gcr.job-id"] == $job
|
||||
and .labels["gcr.run-attempt"] == $attempt
|
||||
and .labels["gcr.label"] == $label
|
||||
and .labels["gcr.location"] == $location
|
||||
and .labels["gcr.arch"] == $arch
|
||||
and ((.server_type.name // .server_type) == $type))]' \
|
||||
"$GCR_LAST_BODY")" || return 2
|
||||
find_count="$(printf '%s' "$find_matches" | jq 'length')" || return 2
|
||||
case "$find_count" in
|
||||
0) return 1 ;;
|
||||
1) printf '%s' "$find_matches" | jq -r '.[0].id' ;;
|
||||
*) return 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
gcr_create_explicitly_rejected() {
|
||||
case "$1" in
|
||||
400|401|403|404|405|409|412|422|423) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
gcr_vm_record_create_ambiguous() {
|
||||
ambiguous_name="$1"; ambiguous_label="$2"; ambiguous_ttl="$3"
|
||||
ambiguous_job="$4"; ambiguous_attempt="$5"; ambiguous_repo="$6"
|
||||
ambiguous_type="$7"; ambiguous_rate="$8"; ambiguous_location="$9"
|
||||
shift 9; ambiguous_arch="$1"; ambiguous_http="$2"; ambiguous_curl="$3"
|
||||
ambiguous_rec="$(jq -n --arg j "$ambiguous_job" --arg a "$ambiguous_attempt" \
|
||||
--arg r "$ambiguous_repo" --arg l "$ambiguous_label" \
|
||||
--arg t "$(gcr_now_epoch)" --arg vn "$ambiguous_name" \
|
||||
--arg ttl "$ambiguous_ttl" --arg st "$ambiguous_type" \
|
||||
--arg rate "$ambiguous_rate" --arg loc "$ambiguous_location" \
|
||||
--arg arch "$ambiguous_arch" --arg http "$ambiguous_http" \
|
||||
--arg curl "$ambiguous_curl" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:($ttl|tonumber), vm_id:"", vm_name:$vn,
|
||||
server_type:$st, budget_rate:$rate, candidate_location:$loc,
|
||||
candidate_arch:$arch, create_http:$http, create_curl_status:$curl,
|
||||
bootstrapped:false,
|
||||
status:"create_ambiguous"}')"
|
||||
gcr_record_put "$ambiguous_job" "$ambiguous_attempt" "$ambiguous_rec"
|
||||
}
|
||||
|
||||
gcr_vm_build_userdata() {
|
||||
vm_name="$1"; label="$2"; reg_token="$3"
|
||||
|
||||
@@ -92,8 +158,8 @@ labels:
|
||||
$GCR_DEBUG_SSH_PUBKEY"
|
||||
fi
|
||||
|
||||
# NOTE(yukkop): token reaches only this VM's Hetzner metadata service;
|
||||
# ephemeral registration makes it useless after the single job exits.
|
||||
# NOTE(yukkop): token reaches only this VM's Hetzner metadata service and
|
||||
# is used for initial registration, not for later idle-slot assignments.
|
||||
printf '%s' "#cloud-config
|
||||
write_files:
|
||||
$ssh_key_block
|
||||
@@ -111,7 +177,7 @@ $(printf '%s\n' "$runner_config" | sed 's/^/ /')
|
||||
- path: /etc/systemd/system/gitea-runner.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Gitea ephemeral Actions runner
|
||||
Description=Gitea on-demand Actions runner
|
||||
After=network-online.target gcr-bootstrap.service
|
||||
Requires=gcr-bootstrap.service
|
||||
|
||||
@@ -119,7 +185,7 @@ $(printf '%s\n' "$runner_config" | sed 's/^/ /')
|
||||
Type=simple
|
||||
Environment=GITEA_INSTANCE_URL=$GCR_GITEA_URL
|
||||
Environment=GITEA_RUNNER_REGISTRATION_TOKEN=$reg_token
|
||||
ExecStart=/usr/local/bin/act_runner daemon --ephemeral --config /etc/gitea-runner/config.yaml
|
||||
ExecStart=/usr/local/bin/act_runner daemon --config /etc/gitea-runner/config.yaml
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -163,9 +229,10 @@ runcmd:
|
||||
}
|
||||
|
||||
# gcr_vm_create NAME LABEL SERVER_TYPE TTL_MIN REG_TOKEN JOB_ID ATTEMPT REPO
|
||||
# Prints new server id.
|
||||
# Caller holds admission lock. Reserves each affordable candidate before its
|
||||
# create request; prints new server id, actual server type, and reserved rate.
|
||||
gcr_vm_create() {
|
||||
vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4"
|
||||
vm_name="$1"; label="$2"; ttl_min="$4"
|
||||
reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8"
|
||||
|
||||
ttl_min="$(gcr_label_ttl "$label")" || return 1
|
||||
@@ -174,8 +241,18 @@ gcr_vm_create() {
|
||||
while read -r candidate_type candidate_loc candidate_arch; do
|
||||
[ -n "${candidate_type:-}" ] || continue
|
||||
candidate_n=$((candidate_n + 1))
|
||||
candidate_rate="$(gcr_server_hourly_rate "$candidate_type")" || continue
|
||||
if ! gcr_budget_can_add "$candidate_rate" "$ttl_min"; then
|
||||
gcr_log info --ns=hcloud "skip candidate[$candidate_n] label=$label type=$candidate_type over budget"
|
||||
continue
|
||||
fi
|
||||
if ! gcr_budget_add "$candidate_rate" "$ttl_min"; then
|
||||
gcr_log error --ns=hcloud "budget reservation write failed label=$label type=$candidate_type"
|
||||
return 1
|
||||
fi
|
||||
image_id="$(gcr_image_id_for_arch "$candidate_arch" "$label")" || {
|
||||
gcr_log warn --ns=hcloud "skip candidate[$candidate_n] label=$label arch=$candidate_arch no image"
|
||||
gcr_budget_sub "$candidate_rate" "$ttl_min" || return 1
|
||||
continue
|
||||
}
|
||||
payload="$(jq -n \
|
||||
@@ -203,9 +280,34 @@ gcr_vm_create() {
|
||||
"gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')"
|
||||
gcr_log info --ns=hcloud "try candidate[$candidate_n] label=$label type=$candidate_type arch=$candidate_arch loc=$candidate_loc"
|
||||
if gcr_hcloud_req POST /servers "$payload"; then
|
||||
jq -r '.server.id' "$GCR_LAST_BODY"
|
||||
printf '%s %s %s\n' \
|
||||
"$(jq -r '.server.id' "$GCR_LAST_BODY")" "$candidate_type" "$candidate_rate"
|
||||
return 0
|
||||
fi
|
||||
create_http="${GCR_LAST_HTTP:-000}"
|
||||
create_curl="${GCR_LAST_CURL_STATUS:-0}"
|
||||
find_status=0
|
||||
found_vm_id="$(gcr_vm_find_created "$vm_name" "$job_id" "$attempt" \
|
||||
"$label" "$candidate_type" "$candidate_loc" "$candidate_arch")" \
|
||||
|| find_status="$?"
|
||||
if [ "$find_status" -eq 0 ]; then
|
||||
printf '%s %s %s\n' "$found_vm_id" "$candidate_type" "$candidate_rate"
|
||||
return 0
|
||||
fi
|
||||
if [ "$find_status" -eq 1 ] \
|
||||
&& gcr_create_explicitly_rejected "$create_http"; then
|
||||
if ! gcr_budget_sub "$candidate_rate" "$ttl_min"; then
|
||||
gcr_log error --ns=hcloud "budget reservation rollback failed label=$label type=$candidate_type"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
if ! gcr_vm_record_create_ambiguous "$vm_name" "$label" "$ttl_min" \
|
||||
"$job_id" "$attempt" "$repo" "$candidate_type" "$candidate_rate" \
|
||||
"$candidate_loc" "$candidate_arch" "$create_http" "$create_curl"; then
|
||||
gcr_log error --ns=hcloud "cannot persist ambiguous create job=$job_id type=$candidate_type"
|
||||
fi
|
||||
return 2
|
||||
fi
|
||||
if [ "$candidate_n" -le 3 ]; then
|
||||
sleep 5
|
||||
else
|
||||
@@ -217,7 +319,7 @@ EOF
|
||||
return 1
|
||||
}
|
||||
|
||||
# gcr_vm_destroy SERVER_ID — idempotent best-effort destroy.
|
||||
# gcr_vm_destroy SERVER_ID — success means DELETE returned HTTP 2xx.
|
||||
gcr_vm_destroy() {
|
||||
if ! gcr_hcloud_req DELETE "/servers/$1"; then
|
||||
gcr_log warn --ns=hcloud "destroy failed or already gone: server $1"
|
||||
@@ -225,6 +327,77 @@ gcr_vm_destroy() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Only cleanup records establish prior ownership, making DELETE 404 a
|
||||
# confirmed-absent success rather than an ambiguous lookup failure.
|
||||
gcr_vm_destroy_owned() {
|
||||
gcr_vm_destroy "$1" && return 0
|
||||
[ "${GCR_LAST_HTTP:-}" = "404" ]
|
||||
}
|
||||
|
||||
# Caller holds the lifecycle path's existing ownership locks.
|
||||
gcr_vm_cleanup_pending() {
|
||||
cleanup_job="$1"; cleanup_attempt="$2"; cleanup_rec="$3"
|
||||
cleanup_vm_id="$(gcr_record_field "$cleanup_rec" vm_id)"
|
||||
if [ "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" != "true" ]; then
|
||||
gcr_vm_destroy_owned "$cleanup_vm_id" || return 1
|
||||
cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_vm_destroyed = true')"
|
||||
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
|
||||
fi
|
||||
if [ "$(gcr_record_field "$cleanup_rec" cleanup_refund_budget)" = "true" ] \
|
||||
&& [ "$(gcr_record_field "$cleanup_rec" cleanup_budget_released)" != "true" ]; then
|
||||
cleanup_rate="$(gcr_record_field "$cleanup_rec" budget_rate)"
|
||||
cleanup_ttl="$(gcr_record_field "$cleanup_rec" ttl_min)"
|
||||
cleanup_refund_key="$(gcr_alloc_key "$cleanup_job" "$cleanup_attempt")"
|
||||
gcr_budget_refund_once "$cleanup_refund_key" "$cleanup_rate" "$cleanup_ttl" \
|
||||
|| return 1
|
||||
cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_budget_released = true')"
|
||||
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
|
||||
fi
|
||||
gcr_record_del "$cleanup_job" "$cleanup_attempt"
|
||||
}
|
||||
|
||||
# Persist intent before DELETE. Normal lifecycle teardown never changes budget;
|
||||
# failed creation passes REFUND_BUDGET=true to release its unused reservation.
|
||||
gcr_vm_cleanup_start() {
|
||||
cleanup_job="$1"; cleanup_attempt="$2"; cleanup_source="$3"
|
||||
cleanup_reason="$4"; cleanup_refund="$5"
|
||||
cleanup_rec="$(printf '%s' "$cleanup_source" | jq -c \
|
||||
--arg reason "$cleanup_reason" --argjson refund "$cleanup_refund" \
|
||||
'.status = "cleanup_pending"
|
||||
| .cleanup_reason = $reason
|
||||
| .cleanup_refund_budget = $refund
|
||||
| .cleanup_vm_destroyed = false
|
||||
| .cleanup_budget_released = false')"
|
||||
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
|
||||
gcr_vm_cleanup_pending "$cleanup_job" "$cleanup_attempt" "$cleanup_rec"
|
||||
}
|
||||
|
||||
# Caller holds allocation and admission locks. A failed primary write first
|
||||
# persists cleanup ownership; reservation is released only after destroy.
|
||||
gcr_vm_record_created() {
|
||||
record_job="$1"; record_attempt="$2"; record_repo="$3"; record_label="$4"
|
||||
record_created="$5"; record_vm_id="$6"; record_vm_name="$7"
|
||||
record_ttl="$8"; record_type="$9"; shift 9; record_rate="$1"
|
||||
record_rec="$(jq -n --arg j "$record_job" --arg a "$record_attempt" \
|
||||
--arg r "$record_repo" --arg l "$record_label" --arg t "$record_created" \
|
||||
--arg v "$record_vm_id" --arg vn "$record_vm_name" --arg ttl "$record_ttl" \
|
||||
--arg st "$record_type" --arg rate "$record_rate" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
|
||||
vm_name:$vn, server_type:$st, budget_rate:$rate,
|
||||
bootstrapped:false, status:"pending_vm"}')"
|
||||
gcr_record_put "$record_job" "$record_attempt" "$record_rec" && return 0
|
||||
|
||||
if ! gcr_vm_cleanup_start "$record_job" "$record_attempt" "$record_rec" \
|
||||
state-write-failed true; then
|
||||
cleanup_rec="$(gcr_record_get "$record_job" "$record_attempt")"
|
||||
[ "$(gcr_record_field "$cleanup_rec" status)" = "cleanup_pending" ] && return 1
|
||||
gcr_log error --ns=alloc "cannot persist cleanup record job=$record_job vm=$record_vm_id"
|
||||
return 1
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_vm_public_ip() {
|
||||
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
|
||||
if gcr_hcloud_req GET "/servers/$1"; then
|
||||
@@ -232,6 +405,31 @@ gcr_vm_public_ip() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop idle runners so Gitea cannot schedule work before atomic reuse claim.
|
||||
# The controller starts the service only after the claim record is written.
|
||||
gcr_vm_runner_service() {
|
||||
vm_id="$1"; action="$2"
|
||||
case "$action" in
|
||||
start|stop) service_command="systemctl $action gitea-runner.service" ;;
|
||||
health) service_command="systemctl is-active --quiet gitea-runner.service" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
ip="$(gcr_vm_public_ip "$vm_id")" || return 1
|
||||
[ -n "$ip" ] || return 1
|
||||
test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || return 1
|
||||
key_tmp="$(mktemp "${TMPDIR:-/tmp}/gcr-runner-sshkey.XXXXXX")"
|
||||
cat "$GCR_SSH_PRIVKEY_FILE" > "$key_tmp"
|
||||
printf '\n' >> "$key_tmp"
|
||||
chmod 0600 "$key_tmp"
|
||||
ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes"
|
||||
if timeout 30 ssh $ssh_opts "root@$ip" "$service_command"; then
|
||||
rm -f "$key_tmp"
|
||||
return 0
|
||||
fi
|
||||
rm -f "$key_tmp"
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_vm_collect_diagnostics() {
|
||||
vm_id="$1"; ip="$2"; job_id="$3"; reason="$4"
|
||||
|
||||
@@ -274,6 +472,76 @@ gcr_vm_collect_diagnostics() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Finish a terminal job under its allocation lock. Healthy bootstrapped VMs
|
||||
# become idle until their existing billing boundary; every unsafe transition
|
||||
# uses durable cleanup_pending teardown instead.
|
||||
gcr_vm_finish_terminal() {
|
||||
finish_job="$1"; finish_attempt="$2"; finish_rec="$3"; finish_state="$4"
|
||||
finish_via="${5:-webhook}"
|
||||
finish_vm_id="$(gcr_record_field "$finish_rec" vm_id)"
|
||||
|
||||
if [ -n "$finish_vm_id" ] && [ "$finish_vm_id" != "null" ] \
|
||||
&& [ "$finish_vm_id" != "0" ]; then
|
||||
case "$finish_state" in
|
||||
completed:success|completed:cancelled|completed:skipped) ;;
|
||||
completed:*)
|
||||
finish_ip="$(gcr_vm_public_ip "$finish_vm_id" || true)"
|
||||
gcr_vm_collect_diagnostics "$finish_vm_id" "$finish_ip" \
|
||||
"$finish_job" "$finish_state" || true
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if finish_idle_rec="$(gcr_record_idle_json "$finish_rec")"; then
|
||||
gcr_lock_acquire idle-pool || return 2
|
||||
finish_repo="$(gcr_record_field "$finish_rec" repo)"
|
||||
finish_runner="$(gcr_record_field "$finish_rec" vm_name)"
|
||||
if ! gcr_vm_runner_service "$finish_vm_id" health; then
|
||||
gcr_lock_release idle-pool
|
||||
finish_cleanup_reason=idle-health-failed
|
||||
elif ! gcr_gitea_runner_disabled "$finish_repo" "$finish_runner" true \
|
||||
|| ! gcr_vm_runner_service "$finish_vm_id" stop; then
|
||||
gcr_lock_release idle-pool
|
||||
finish_cleanup_reason=idle-stop-failed
|
||||
elif ! gcr_record_put "$finish_job" "$finish_attempt" "$finish_idle_rec"; then
|
||||
gcr_lock_release idle-pool
|
||||
finish_cleanup_reason=idle-state-write-failed
|
||||
else
|
||||
finish_expires="$(gcr_record_field "$finish_idle_rec" idle_expires_at)"
|
||||
gcr_lock_release idle-pool
|
||||
gcr_event "vm-idle" "$finish_job" \
|
||||
"{\"vm_id\":$finish_vm_id,\"expires_at\":$finish_expires,\"via\":\"$finish_via\"}"
|
||||
gcr_log info --ns=sweep \
|
||||
"job=$finish_job terminal ($finish_state), retaining vm=$finish_vm_id until $finish_expires"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if gcr_vm_cleanup_start "$finish_job" "$finish_attempt" "$finish_rec" \
|
||||
"$finish_cleanup_reason" false; then
|
||||
gcr_event "vm-destroyed" "$finish_job" \
|
||||
"{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_cleanup_reason\",\"via\":\"$finish_via\"}"
|
||||
else
|
||||
gcr_event "vm-cleanup-pending" "$finish_job" \
|
||||
"{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_cleanup_reason\",\"via\":\"$finish_via\"}"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -n "$finish_vm_id" ] && [ "$finish_vm_id" != "null" ] \
|
||||
&& [ "$finish_vm_id" != "0" ]; then
|
||||
if gcr_vm_cleanup_start "$finish_job" "$finish_attempt" "$finish_rec" \
|
||||
"$finish_state" false; then
|
||||
gcr_event "vm-destroyed" "$finish_job" \
|
||||
"{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_state\",\"via\":\"$finish_via\"}"
|
||||
else
|
||||
gcr_event "vm-cleanup-pending" "$finish_job" \
|
||||
"{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_state\",\"via\":\"$finish_via\"}"
|
||||
fi
|
||||
else
|
||||
gcr_record_del "$finish_job" "$finish_attempt"
|
||||
fi
|
||||
}
|
||||
|
||||
# Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's
|
||||
# cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the
|
||||
# controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose
|
||||
@@ -321,7 +589,7 @@ STARTEOF
|
||||
chmod 0700 /usr/local/sbin/gcr-runner-start
|
||||
cat > /etc/systemd/system/gitea-runner.service <<UNITEOF
|
||||
[Unit]
|
||||
Description=Gitea ephemeral Actions runner
|
||||
Description=Gitea on-demand Actions runner
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
|
||||
@@ -20,9 +20,54 @@ gcr_record_path() {
|
||||
printf '%s/jobs/%s.json' "$GCR_STATE_DIR" "$(gcr_alloc_key "$1" "$2")"
|
||||
}
|
||||
|
||||
# mkdir(2) atomicity guard: succeeds exactly once per key until released.
|
||||
# mkdir(2) atomicity guard. Owner metadata lets a new controller process
|
||||
# recover locks stranded by a crashed webhook or reconciler process.
|
||||
gcr_lock_takeover() {
|
||||
gcr_takeover_dir="$1"
|
||||
gcr_takeover_old="$gcr_takeover_dir.reclaim.$$"
|
||||
# Rename is atomic: exactly one reclaimer can move the observed stale
|
||||
# directory. Never rm -rf the active lock pathname during recovery.
|
||||
mv "$gcr_takeover_dir" "$gcr_takeover_old" 2>/dev/null || return 1
|
||||
if mkdir "$gcr_takeover_dir" 2>/dev/null; then
|
||||
printf '%s %s\n' "$$" "$(gcr_now_epoch)" > "$gcr_takeover_dir/owner"
|
||||
rm -rf "$gcr_takeover_old"
|
||||
return 0
|
||||
fi
|
||||
rm -rf "$gcr_takeover_old"
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_lock_acquire() {
|
||||
mkdir "$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$1")" 2>/dev/null
|
||||
gcr_lock_key="$1"
|
||||
gcr_lock_dir="$(printf '%s/jobs/.lock.%s' "$GCR_STATE_DIR" "$gcr_lock_key")"
|
||||
if mkdir "$gcr_lock_dir" 2>/dev/null; then
|
||||
printf '%s %s\n' "$$" "$(gcr_now_epoch)" > "$gcr_lock_dir/owner"
|
||||
return 0
|
||||
fi
|
||||
|
||||
gcr_lock_owner="$(cat "$gcr_lock_dir/owner" 2>/dev/null || true)"
|
||||
if [ -z "$gcr_lock_owner" ]; then
|
||||
# A crash between mkdir and owner write leaves no PID. Give a live
|
||||
# creator a short initialization window, then unblock hard TTL work.
|
||||
gcr_lock_mtime="$(stat -c %Y "$gcr_lock_dir" 2>/dev/null || true)"
|
||||
gcr_lock_now="$(gcr_now_epoch)"
|
||||
case "$gcr_lock_mtime:$gcr_lock_now" in
|
||||
*[!0-9:]*|:*|*::*|*:) return 1 ;;
|
||||
esac
|
||||
[ "$((gcr_lock_now - gcr_lock_mtime))" -ge 30 ] || return 1
|
||||
gcr_lock_takeover "$gcr_lock_dir"
|
||||
return "$?"
|
||||
fi
|
||||
set -- $gcr_lock_owner
|
||||
gcr_lock_pid="${1:-}"
|
||||
case "$gcr_lock_pid" in ''|*[!0-9]*) return 1 ;; esac
|
||||
if kill -0 "$gcr_lock_pid" 2>/dev/null; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Dead PID means a process crash, not live contention. Atomically take
|
||||
# over its directory; concurrent recovery cannot erase a new lock.
|
||||
gcr_lock_takeover "$gcr_lock_dir"
|
||||
}
|
||||
|
||||
gcr_lock_release() {
|
||||
@@ -46,7 +91,151 @@ gcr_record_del() {
|
||||
}
|
||||
|
||||
gcr_record_field() {
|
||||
printf '%s' "$1" | jq -r --arg f "$2" '.[$f] // ""'
|
||||
printf '%s' "$1" | jq -r --arg f "$2" 'if has($f) then .[$f] else "" end'
|
||||
}
|
||||
|
||||
gcr_now_epoch() {
|
||||
date -u '+%s'
|
||||
}
|
||||
|
||||
# Healthy bootstrapped VMs remain reusable until next billing-hour boundary,
|
||||
# but never beyond profile hard TTL. Prints updated idle record when safe.
|
||||
gcr_record_idle_json() {
|
||||
gcr_idle_rec="$1"
|
||||
[ "$(gcr_record_field "$gcr_idle_rec" bootstrapped)" = "true" ] || return 1
|
||||
|
||||
gcr_idle_vm_id="$(gcr_record_field "$gcr_idle_rec" vm_id)"
|
||||
gcr_idle_created="$(gcr_record_field "$gcr_idle_rec" created_at)"
|
||||
gcr_idle_ttl="$(gcr_record_field "$gcr_idle_rec" ttl_min)"
|
||||
case "$gcr_idle_vm_id:$gcr_idle_created:$gcr_idle_ttl" in
|
||||
*[!0-9:]*|0:*|:*|*::*|*:) return 1 ;;
|
||||
esac
|
||||
|
||||
gcr_idle_now="$(gcr_now_epoch)"
|
||||
gcr_idle_hard_expires="$((gcr_idle_created + gcr_idle_ttl * 60))"
|
||||
[ "$gcr_idle_now" -lt "$gcr_idle_hard_expires" ] || return 1
|
||||
|
||||
gcr_idle_age="$((gcr_idle_now - gcr_idle_created))"
|
||||
[ "$gcr_idle_age" -ge 0 ] || gcr_idle_age=0
|
||||
gcr_idle_slots="$((gcr_idle_age / 3600))"
|
||||
[ "$((gcr_idle_age % 3600))" -eq 0 ] || gcr_idle_slots=$((gcr_idle_slots + 1))
|
||||
[ "$gcr_idle_slots" -gt 0 ] || gcr_idle_slots=1
|
||||
gcr_idle_expires="$((gcr_idle_created + gcr_idle_slots * 3600))"
|
||||
[ "$gcr_idle_expires" -le "$gcr_idle_hard_expires" ] \
|
||||
|| gcr_idle_expires="$gcr_idle_hard_expires"
|
||||
|
||||
printf '%s' "$gcr_idle_rec" | jq -c \
|
||||
--arg now "$gcr_idle_now" --arg expires "$gcr_idle_expires" \
|
||||
'.status = "idle_vm"
|
||||
| .idle_since = ($now | tonumber)
|
||||
| .idle_expires_at = ($expires | tonumber)'
|
||||
}
|
||||
|
||||
gcr_idle_record_unexpired() {
|
||||
gcr_idle_rec="$1"
|
||||
[ "$(gcr_record_field "$gcr_idle_rec" status)" = "idle_vm" ] || return 1
|
||||
[ "$(gcr_record_field "$gcr_idle_rec" bootstrapped)" = "true" ] || return 1
|
||||
|
||||
gcr_idle_now="$(gcr_now_epoch)"
|
||||
gcr_idle_vm_id="$(gcr_record_field "$gcr_idle_rec" vm_id)"
|
||||
gcr_idle_vm_name="$(gcr_record_field "$gcr_idle_rec" vm_name)"
|
||||
gcr_idle_expires="$(gcr_record_field "$gcr_idle_rec" idle_expires_at)"
|
||||
gcr_idle_created="$(gcr_record_field "$gcr_idle_rec" created_at)"
|
||||
gcr_idle_ttl="$(gcr_record_field "$gcr_idle_rec" ttl_min)"
|
||||
case "$gcr_idle_expires:$gcr_idle_created:$gcr_idle_ttl" in
|
||||
*[!0-9:]*|:*|*::*|*:) return 1 ;;
|
||||
esac
|
||||
case "$gcr_idle_vm_id" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ -n "$gcr_idle_vm_name" ] || return 1
|
||||
gcr_idle_hard_expires="$((gcr_idle_created + gcr_idle_ttl * 60))"
|
||||
[ "$gcr_idle_now" -lt "$gcr_idle_expires" ] \
|
||||
&& [ "$gcr_idle_now" -lt "$gcr_idle_hard_expires" ]
|
||||
}
|
||||
|
||||
gcr_idle_record_usable() {
|
||||
gcr_idle_rec="$1"
|
||||
gcr_idle_record_unexpired "$gcr_idle_rec" || return 1
|
||||
gcr_idle_min_remaining="${GCR_RECONCILE_INTERVAL_SEC:-60}"
|
||||
case "$gcr_idle_min_remaining" in ''|*[!0-9]*) return 1 ;; esac
|
||||
[ "$((gcr_idle_expires - gcr_idle_now))" -ge "$gcr_idle_min_remaining" ] \
|
||||
&& [ "$((gcr_idle_hard_expires - gcr_idle_now))" -ge "$gcr_idle_min_remaining" ]
|
||||
}
|
||||
|
||||
# Caller must hold destination allocation lock. Global pool lock ensures one
|
||||
# queued job claims an idle VM; destination write precedes source deletion so
|
||||
# orphan/stale sweeps always see an owner during transfer.
|
||||
gcr_claim_idle() {
|
||||
gcr_claim_job="$1"; gcr_claim_attempt="$2"
|
||||
gcr_claim_repo="$3"; gcr_claim_label="$4"
|
||||
# Exit 2 means pool is busy; callers must defer instead of charging for a
|
||||
# new VM without knowing whether matching paid capacity is available.
|
||||
gcr_lock_acquire idle-pool || return 2
|
||||
|
||||
for gcr_claim_file in $(gcr_active_records); do
|
||||
gcr_claim_rec="$(cat "$gcr_claim_file")"
|
||||
[ "$(gcr_record_field "$gcr_claim_rec" status)" = "idle_vm" ] || continue
|
||||
[ "$(gcr_record_field "$gcr_claim_rec" repo)" = "$gcr_claim_repo" ] || continue
|
||||
[ "$(gcr_record_field "$gcr_claim_rec" label)" = "$gcr_claim_label" ] || continue
|
||||
gcr_idle_record_usable "$gcr_claim_rec" || continue
|
||||
|
||||
gcr_claim_old_job="$(gcr_record_field "$gcr_claim_rec" job_id)"
|
||||
gcr_claim_old_attempt="$(gcr_record_field "$gcr_claim_rec" run_attempt)"
|
||||
gcr_claim_vm_id="$(gcr_record_field "$gcr_claim_rec" vm_id)"
|
||||
gcr_record_vm_owned_elsewhere "$gcr_claim_vm_id" \
|
||||
"$gcr_claim_old_job" "$gcr_claim_old_attempt" && continue
|
||||
gcr_claim_now="$(gcr_now_epoch)"
|
||||
gcr_claim_new="$(printf '%s' "$gcr_claim_rec" | jq -c \
|
||||
--arg job "$gcr_claim_job" --arg attempt "$gcr_claim_attempt" \
|
||||
--arg repo "$gcr_claim_repo" --arg label "$gcr_claim_label" \
|
||||
--arg now "$gcr_claim_now" \
|
||||
'.job_id = $job | .run_attempt = $attempt
|
||||
| .repo = $repo | .label = $label | .status = "pending_vm"
|
||||
| .bootstrapped = false
|
||||
| .reused_vm = true
|
||||
| .assigned_at = ($now | tonumber)
|
||||
| del(.idle_since, .idle_expires_at)')"
|
||||
if ! gcr_record_put "$gcr_claim_job" "$gcr_claim_attempt" "$gcr_claim_new"; then
|
||||
gcr_lock_release idle-pool
|
||||
return 1
|
||||
fi
|
||||
gcr_record_del "$gcr_claim_old_job" "$gcr_claim_old_attempt"
|
||||
gcr_lock_release idle-pool
|
||||
return 0
|
||||
done
|
||||
|
||||
gcr_lock_release idle-pool
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_record_exists_for_vm_id() {
|
||||
gcr_lookup="$1"
|
||||
for gcr_lookup_file in $(gcr_active_records); do
|
||||
[ "$(gcr_record_field "$(cat "$gcr_lookup_file")" vm_id)" = "$gcr_lookup" ] \
|
||||
&& return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_record_exists_for_vm_name() {
|
||||
gcr_lookup="$1"
|
||||
for gcr_lookup_file in $(gcr_active_records); do
|
||||
[ "$(gcr_record_field "$(cat "$gcr_lookup_file")" vm_name)" = "$gcr_lookup" ] \
|
||||
&& return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_record_vm_owned_elsewhere() {
|
||||
gcr_lookup_vm="$1"; gcr_lookup_job="$2"; gcr_lookup_attempt="$3"
|
||||
for gcr_lookup_file in $(gcr_active_records); do
|
||||
gcr_lookup_rec="$(cat "$gcr_lookup_file")"
|
||||
[ "$(gcr_record_field "$gcr_lookup_rec" vm_id)" = "$gcr_lookup_vm" ] || continue
|
||||
if [ "$(gcr_record_field "$gcr_lookup_rec" job_id)" != "$gcr_lookup_job" ] \
|
||||
|| [ "$(gcr_record_field "$gcr_lookup_rec" run_attempt)" != "$gcr_lookup_attempt" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_event() {
|
||||
@@ -55,8 +244,8 @@ gcr_event() {
|
||||
"$(printf '%s' "$3" | jq -Rs .)" >> "$GCR_STATE_DIR/events.jsonl"
|
||||
}
|
||||
|
||||
# Exit-code contract: 0 = recorded under budget, 1 = would exceed cap.
|
||||
gcr_budget_add() {
|
||||
# Exit-code contract: 0 = remains under budget, 1 = would exceed cap.
|
||||
gcr_budget_can_add() {
|
||||
rate="$1"; ttl_min="$2"
|
||||
month="$(date -u '+%Y-%m')"
|
||||
file="$GCR_STATE_DIR/budget/$month"
|
||||
@@ -65,10 +254,71 @@ gcr_budget_add() {
|
||||
if awk -v p="$projected" -v b="${GCR_BUDGET_EUR_MONTHLY:-15}" 'BEGIN {exit !(p > b)}'; then
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$projected" > "$file"
|
||||
return 0
|
||||
}
|
||||
|
||||
gcr_active_records() {
|
||||
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|deferred)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
|
||||
# Atomic replacement preserves last valid total when a write fails.
|
||||
gcr_budget_write() {
|
||||
gcr_budget_file="$1"; gcr_budget_value="$2"
|
||||
gcr_budget_tmp="$(mktemp "$(dirname "$gcr_budget_file")/.budget.XXXXXX")" \
|
||||
|| return 1
|
||||
if ! printf '%s\n' "$gcr_budget_value" > "$gcr_budget_tmp"; then
|
||||
rm -f "$gcr_budget_tmp"
|
||||
return 1
|
||||
fi
|
||||
if ! mv -f "$gcr_budget_tmp" "$gcr_budget_file"; then
|
||||
rm -f "$gcr_budget_tmp"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Caller holds admission lock and has already checked gcr_budget_can_add.
|
||||
gcr_budget_add() {
|
||||
rate="$1"; ttl_min="$2"
|
||||
month="$(date -u '+%Y-%m')"
|
||||
file="$GCR_STATE_DIR/budget/$month"
|
||||
current="$(cat "$file" 2>/dev/null || echo 0)"
|
||||
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')"
|
||||
gcr_budget_write "$file" "$projected"
|
||||
}
|
||||
|
||||
# Caller holds admission lock and is rolling back a matching budget addition.
|
||||
gcr_budget_sub() {
|
||||
rate="$1"; ttl_min="$2"
|
||||
month="$(date -u '+%Y-%m')"
|
||||
file="$GCR_STATE_DIR/budget/$month"
|
||||
current="$(cat "$file" 2>/dev/null || echo 0)"
|
||||
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" \
|
||||
'BEGIN {v = c - r * t / 60; if (v < 0) v = 0; printf "%.4f", v}')"
|
||||
gcr_budget_write "$file" "$projected"
|
||||
}
|
||||
|
||||
# Claim is durable before credit. Existing claim means credit is consumed:
|
||||
# it may have completed, or it may have leaked fail-closed after a crash.
|
||||
# Never subtract twice when outcome between aggregate and state writes is unknown.
|
||||
gcr_budget_refund_once() {
|
||||
refund_key="$1"; refund_rate="$2"; refund_ttl="$3"
|
||||
refund_month="$(date -u '+%Y-%m')"
|
||||
refund_root="$GCR_STATE_DIR/budget/refunds/$refund_month"
|
||||
refund_claim="$refund_root/$refund_key"
|
||||
mkdir -p "$refund_root" || return 1
|
||||
if ! mkdir "$refund_claim" 2>/dev/null; then
|
||||
if [ -d "$refund_claim" ]; then
|
||||
if [ ! -f "$refund_claim/status" ]; then
|
||||
gcr_log error --ns=budget \
|
||||
"refund outcome uncertain key=$refund_key; retaining fail-closed claim"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
if ! printf '%s %s\n' "$refund_rate" "$refund_ttl" > "$refund_claim/intent"; then
|
||||
return 1
|
||||
fi
|
||||
gcr_budget_sub "$refund_rate" "$refund_ttl" || return 1
|
||||
printf 'refunded\n' > "$refund_claim/status" || return 1
|
||||
}
|
||||
|
||||
gcr_active_records() {
|
||||
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred|cleanup_pending|create_ambiguous)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
|
||||
}
|
||||
|
||||
@@ -94,6 +94,12 @@ gcr_alloc() {
|
||||
RESPONSE_CODE=204
|
||||
return 0
|
||||
fi
|
||||
existing="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ -n "$existing" ]; then
|
||||
gcr_lock_release "$key"
|
||||
RESPONSE_CODE=204
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$label_count" -ne 1 ]; then
|
||||
gcr_lock_release "$key"
|
||||
@@ -110,34 +116,77 @@ gcr_alloc() {
|
||||
fi
|
||||
|
||||
set -- $profile
|
||||
server_type="$1"; ttl_min="$2"; rate="$3"
|
||||
server_type="$1"; ttl_min="$2"
|
||||
|
||||
if ! gcr_lock_acquire admission; then
|
||||
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
|
||||
--arg l "$label" --arg t "$(gcr_now_epoch)" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
|
||||
status:"deferred"}')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "deferred" "$job_id" "{\"reason\":\"admission-busy\"}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="deferred: admission busy"
|
||||
return 0
|
||||
fi
|
||||
|
||||
active="$(gcr_count_active)"
|
||||
repo_active="$(gcr_count_active_repo "$repo")"
|
||||
if [ "$active" -ge "${GCR_CONCURRENCY_CAP:-2}" ] \
|
||||
|| [ "$repo_active" -ge "${GCR_PER_REPO_CAP:-1}" ]; then
|
||||
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
|
||||
--arg l "$label" --arg t "$(date -u '+%s')" \
|
||||
--arg l "$label" --arg t "$(gcr_now_epoch)" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
|
||||
status:"deferred"}')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "deferred" "$job_id" "{\"active\":$active,\"repo_active\":$repo_active}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="deferred: capacity"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! gcr_budget_add "$rate" "$ttl_min"; then
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
claim_status=0
|
||||
gcr_claim_idle "$job_id" "$attempt" "$repo" "$label" || claim_status="$?"
|
||||
if [ "$claim_status" -eq 0 ]; then
|
||||
reused="$(gcr_record_get "$job_id" "$attempt")"
|
||||
vm_id="$(gcr_record_field "$reused" vm_id)"
|
||||
vm_name="$(gcr_record_field "$reused" vm_name)"
|
||||
if gcr_vm_runner_service "$vm_id" start \
|
||||
&& gcr_vm_runner_service "$vm_id" health \
|
||||
&& gcr_gitea_runner_disabled "$repo" "$vm_name" false; then
|
||||
reused="$(gcr_record_get "$job_id" "$attempt")"
|
||||
reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')"
|
||||
gcr_record_put "$job_id" "$attempt" "$reused"
|
||||
else
|
||||
gcr_gitea_runner_disabled "$repo" "$vm_name" true || true
|
||||
gcr_event "vm-reuse-start-failed" "$job_id" "{\"vm_id\":$vm_id}"
|
||||
fi
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted"
|
||||
gcr_event "vm-reused" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\"}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="reused $vm_name"
|
||||
return 0
|
||||
fi
|
||||
if [ "$claim_status" -eq 2 ]; then
|
||||
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
|
||||
--arg l "$label" --arg t "$(gcr_now_epoch)" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:null, vm_id:"", vm_name:"",
|
||||
status:"deferred"}')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "deferred" "$job_id" "{\"reason\":\"idle-pool-busy\"}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="deferred: idle pool busy"
|
||||
return 0
|
||||
fi
|
||||
|
||||
reg_token="$(gcr_gitea_registration_token "$repo")" || {
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "token-error" "$job_id" "{}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="registration token unavailable"
|
||||
@@ -145,22 +194,34 @@ gcr_alloc() {
|
||||
}
|
||||
|
||||
vm_name="gcr-${job_id}-${attempt}"
|
||||
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
|
||||
"$reg_token" "$job_id" "$attempt" "$repo")" || {
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
created_at="$(gcr_now_epoch)"
|
||||
create_status=0
|
||||
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
|
||||
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
|
||||
if [ "$create_status" -ne 0 ]; then
|
||||
if [ "$create_status" -eq 2 ]; then
|
||||
gcr_event "vm-create-ambiguous" "$job_id" "{}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="VM creation pending recovery"
|
||||
else
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_event "vm-create-failed" "$job_id" "{}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
|
||||
fi
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "vm-create-failed" "$job_id" "{}"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
|
||||
return 0
|
||||
}
|
||||
|
||||
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
|
||||
--arg l "$label" --arg t "$(date -u '+%s')" --arg v "$vm_id" \
|
||||
--arg vn "$vm_name" --arg ttl "$ttl_min" \
|
||||
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
|
||||
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
|
||||
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
fi
|
||||
set -- $created
|
||||
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
|
||||
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
|
||||
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
|
||||
"$actual_server_type" "$actual_rate"; then
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
RESPONSE_CODE=202; RESPONSE_BODY="VM state write failed"
|
||||
return 0
|
||||
fi
|
||||
gcr_lock_release admission
|
||||
gcr_lock_release "$key"
|
||||
gcr_event "vm-created" "$job_id" "{\"vm_id\":$vm_id,\"label\":\"$label\",\"ttl_min\":$ttl_min}"
|
||||
|
||||
@@ -170,24 +231,45 @@ gcr_alloc() {
|
||||
gcr_deallocate() {
|
||||
job_id="$1"; attempt="$2"; new_status="$3"
|
||||
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
[ -n "$rec" ] || return 0
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || return 0
|
||||
|
||||
vm_id="$(gcr_record_field "$rec" vm_id)"
|
||||
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
|
||||
case "$new_status" in
|
||||
completed:success|completed:cancelled|completed:skipped) ;;
|
||||
completed:*)
|
||||
ip="$(gcr_vm_public_ip "$vm_id" || true)"
|
||||
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true
|
||||
;;
|
||||
esac
|
||||
gcr_vm_destroy "$vm_id" || true
|
||||
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ -z "$rec" ]; then
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
fi
|
||||
|
||||
gcr_record_del "$job_id" "$attempt"
|
||||
gcr_lock_release "$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
case "$(gcr_record_field "$rec" status)" in
|
||||
pending_vm|vm_active) ;;
|
||||
*)
|
||||
gcr_lock_release "$key"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
finish_status=0
|
||||
gcr_vm_finish_terminal "$job_id" "$attempt" "$rec" "$new_status" webhook \
|
||||
|| finish_status="$?"
|
||||
gcr_lock_release "$key"
|
||||
case "$finish_status" in
|
||||
0|2) return 0 ;;
|
||||
*) return "$finish_status" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
gcr_mark_in_progress() {
|
||||
job_id="$1"; attempt="$2"
|
||||
key="$(gcr_alloc_key "$job_id" "$attempt")"
|
||||
gcr_lock_acquire "$key" || return 0
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
case "$(gcr_record_field "$rec" status)" in
|
||||
pending_vm|vm_active)
|
||||
rec="$(printf '%s' "$rec" | jq -c '.status = "vm_active"')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
;;
|
||||
esac
|
||||
gcr_lock_release "$key"
|
||||
}
|
||||
|
||||
gcr_handle_webhook() {
|
||||
@@ -210,8 +292,11 @@ gcr_handle_webhook() {
|
||||
repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')"
|
||||
labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')"
|
||||
|
||||
case "$action:$job_id" in
|
||||
:*|"queued:"|*":0") gcr_respond 400 "malformed payload"; exit 0 ;;
|
||||
[ -n "$action" ] || { gcr_respond 400 "malformed payload"; exit 0; }
|
||||
case "$job_id:$attempt" in
|
||||
*[!0-9:]*|:*|*::*|*:|0:*)
|
||||
gcr_respond 400 "malformed payload"; exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$action" in
|
||||
@@ -220,11 +305,7 @@ gcr_handle_webhook() {
|
||||
gcr_log info --ns=alloc "queued job=$job_id repo=$repo code=$RESPONSE_CODE $RESPONSE_BODY"
|
||||
;;
|
||||
in_progress)
|
||||
rec="$(gcr_record_get "$job_id" "$attempt")"
|
||||
if [ -n "$rec" ]; then
|
||||
rec="$(printf '%s' "$rec" | jq -c '.status = "vm_active"')"
|
||||
gcr_record_put "$job_id" "$attempt" "$rec"
|
||||
fi
|
||||
gcr_mark_in_progress "$job_id" "$attempt"
|
||||
RESPONSE_CODE=204
|
||||
;;
|
||||
completed)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
...
|
||||
}: let
|
||||
src = ./.;
|
||||
cargo = cargoToml src;
|
||||
cargo = cargoToml ./Cargo.toml;
|
||||
in
|
||||
pkgs.rustPlatform.buildRustPackage {
|
||||
pname = cargo.package.name;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
...
|
||||
}: let
|
||||
src = ./.;
|
||||
cargo = cargoToml src;
|
||||
cargo = cargoToml ./Cargo.toml;
|
||||
in
|
||||
pkgs.rustPlatform.buildRustPackage {
|
||||
pname = cargo.package.name;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
...
|
||||
}: let
|
||||
src = ./.;
|
||||
cargo = cargoToml src;
|
||||
cargo = cargoToml ./Cargo.toml;
|
||||
in
|
||||
pkgs.rustPlatform.buildRustPackage {
|
||||
pname = cargo.package.name;
|
||||
|
||||
Executable
+136
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a Prism auto-update instance and immutable packwiz release from an mrpack.
|
||||
|
||||
Upload the pack directory to an immutable release, then atomically switch current.
|
||||
Publish only releases tested with the server, including Minecraft/NeoForge upgrades.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import html
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import struct
|
||||
import tomllib
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
BOOTSTRAP_URL = 'https://github.com/packwiz/packwiz-installer-bootstrap/releases/download/v0.0.3/packwiz-installer-bootstrap.jar'
|
||||
BOOTSTRAP_SHA256 = 'a8fbb24dc604278e97f4688e82d3d91a318b98efc08d5dbfcbcbcab6443d116c'
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
def quote(s):
|
||||
return json.dumps(s, ensure_ascii=False)
|
||||
|
||||
def safe_path(s):
|
||||
p = PurePosixPath(s)
|
||||
if not s or p.is_absolute() or any(x in ('', '.', '..') for x in s.split('/')) or re.search(r'[\\\x00-\x1f:*?"<>|]', s):
|
||||
raise ValueError(f'Unsafe pack path: {s!r}')
|
||||
return p
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('mrpack', type=Path)
|
||||
ap.add_argument('output', type=Path, help='new output directory (must not exist)')
|
||||
ap.add_argument('--base-url', default='https://store.hectic-lab.com/minecraft/world-of-sosal/')
|
||||
ap.add_argument('--server', help='optional initial multiplayer server address')
|
||||
ap.add_argument('--bootstrap', type=Path)
|
||||
args = ap.parse_args()
|
||||
base = args.base_url.rstrip('/') + '/'
|
||||
if not base.startswith('https://'):
|
||||
ap.error('--base-url must use HTTPS')
|
||||
archive = args.mrpack.read_bytes()
|
||||
release = digest(archive)
|
||||
bootstrap = args.bootstrap.read_bytes() if args.bootstrap else urllib.request.urlopen(BOOTSTRAP_URL, timeout=60).read()
|
||||
if digest(bootstrap) != BOOTSTRAP_SHA256:
|
||||
raise ValueError('Bootstrap checksum mismatch')
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
root = args.output / 'pack'
|
||||
root.mkdir(parents=True)
|
||||
entries = {}
|
||||
destinations = set()
|
||||
def write(path, data, *, metafile=False, preserve=False):
|
||||
safe_path(path)
|
||||
dest = root / path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(data)
|
||||
entries[path] = {'file': path, 'hash': digest(data), 'metafile': metafile, 'preserve': preserve}
|
||||
with zipfile.ZipFile(io.BytesIO(archive)) as z:
|
||||
manifest = json.loads(z.read('modrinth.index.json'))
|
||||
deps = manifest['dependencies']
|
||||
if manifest['formatVersion'] != 1 or manifest['game'] != 'minecraft' or set(deps) != {'minecraft', 'neoforge'}:
|
||||
raise ValueError('Expected a Minecraft NeoForge mrpack v1')
|
||||
for f in manifest['files']:
|
||||
path = safe_path(f['path'])
|
||||
if f.get('env', {}).get('client') == 'unsupported':
|
||||
continue
|
||||
if f['path'] in destinations:
|
||||
raise ValueError('Duplicate destination: ' + f['path'])
|
||||
destinations.add(f['path'])
|
||||
url = f['downloads'][0]
|
||||
sha = f['hashes']['sha512']
|
||||
if not url.startswith('https://') or not re.fullmatch('[0-9a-fA-F]{128}', sha):
|
||||
raise ValueError('Invalid URL/hash: ' + f['path'])
|
||||
# Include optional mods too, matching the server importer. The published
|
||||
# manifest, rather than upstream latest versions, controls all updates.
|
||||
meta = f'name = {quote(path.name)}\nfilename = {quote(path.name)}\nside = "client"\n\n[download]\nurl = {quote(url)}\nhash-format = "sha512"\nhash = {quote(sha)}\n'
|
||||
write(str(path) + '.pw.toml', meta.encode(), metafile=True)
|
||||
for prefix in ('overrides/', 'client-overrides/'):
|
||||
for item in z.infolist():
|
||||
if not item.filename.startswith(prefix) or item.is_dir():
|
||||
continue
|
||||
name = item.filename[len(prefix):]
|
||||
safe_path(name)
|
||||
if not (name.startswith(('config/', 'mods/', 'resourcepacks/', 'shaderpacks/')) or name == 'options.txt'):
|
||||
raise ValueError('Review unexpected override: ' + name)
|
||||
if name in destinations:
|
||||
raise ValueError('Override duplicates downloaded mod: ' + name)
|
||||
write(name, z.read(item), preserve=name == 'options.txt')
|
||||
index = 'hash-format = "sha256"\n'
|
||||
for entry in sorted(entries.values(), key=lambda e: e['file']):
|
||||
index += '\n[[files]]\n'
|
||||
for k, v in entry.items():
|
||||
if isinstance(v, bool):
|
||||
if v:
|
||||
index += f'{k} = true\n'
|
||||
else:
|
||||
index += f'{k} = {quote(v)}\n'
|
||||
(root / 'index.toml').write_text(index)
|
||||
pack = f'name = "WorldOfSosal"\npack-format = "packwiz:1.1.0"\nversion = {quote(manifest["versionId"])}\n\n[index]\nfile = "index.toml"\nhash-format = "sha256"\nhash = "{digest(index.encode())}"\n\n[versions]\nminecraft = {quote(deps["minecraft"])}\nneoforge = {quote(deps["neoforge"])}\n'
|
||||
tomllib.loads(pack)
|
||||
(root / 'pack.toml').write_text(pack)
|
||||
(args.output / 'latest.mrpack').write_bytes(archive)
|
||||
# Prism otherwise uses its legacy INI parser and corrupts quoted commands.
|
||||
cfg = '\n'.join(['[General]', 'ConfigVersion=1.2', 'InstanceType=OneSix', 'name=WorldOfSosal Auto Update', 'iconKey=default', 'OverrideCommands=true', 'PreLaunchCommand=' + quote(f'"$INST_JAVA" -jar packwiz-installer-bootstrap.jar {base}current/pack.toml'), 'OverrideMemory=true', 'MinMemAlloc=1024', 'MaxMemAlloc=8192', ''])
|
||||
mmc = {'formatVersion': 1, 'components': [{'uid':'net.minecraft', 'version':deps['minecraft'], 'important':True}, {'uid':'net.neoforged', 'version':deps['neoforge'], 'important':True}]}
|
||||
def nbt_string(value):
|
||||
data = value.encode()
|
||||
return struct.pack('>H', len(data)) + data
|
||||
# Initial server list is seeded only during instance import, never overwritten.
|
||||
servers = b'\x0a\x00\x00\x09' + nbt_string('servers') + b'\x0a\x00\x00\x00\x01'
|
||||
servers += b'\x08' + nbt_string('name') + nbt_string('WorldOfSosal')
|
||||
servers += b'\x08' + nbt_string('ip') + nbt_string(args.server or '') + b'\x00\x00'
|
||||
with zipfile.ZipFile(args.output / 'WorldOfSosal-Prism.zip', 'w', zipfile.ZIP_DEFLATED) as z:
|
||||
z.writestr('instance.cfg', cfg)
|
||||
z.writestr('mmc-pack.json', json.dumps(mmc, indent=2))
|
||||
z.writestr('.minecraft/packwiz-installer-bootstrap.jar', bootstrap)
|
||||
if args.server:
|
||||
z.writestr('.minecraft/servers.dat', servers)
|
||||
page = f'''<!doctype html><html lang="ru"><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>WorldOfSosal</title>
|
||||
<h1>WorldOfSosal — автоматическое обновление</h1>
|
||||
<p>Один раз скачайте <a href="WorldOfSosal-Prism.zip">инстанс для Prism Launcher</a>, затем выберите «Добавить сборку → Импорт из zip». Разрешите команду перед запуском: она устанавливает и обновляет моды через packwiz.</p>
|
||||
<p>Minecraft {html.escape(deps['minecraft'])} · NeoForge {html.escape(deps['neoforge'])} · Java 21 · память 8 ГБ.</p>
|
||||
<p>{'Сервер: <code>' + html.escape(args.server) + '</code>.' if args.server else 'Адрес игрового сервера будет сообщён отдельно.'}</p>
|
||||
<p>Моды и конфигурация сборки обновляются при каждом запуске. Личные настройки options.txt сохраняются. При смене Minecraft или NeoForge может потребоваться повторный запуск после обновления версии.</p>
|
||||
<p><a href="latest.mrpack">Обычный mrpack без автообновлений</a> · <a href="SHA256SUMS">Контрольные суммы</a></p></html>'''
|
||||
(args.output / 'index.html').write_text(page)
|
||||
files = ['WorldOfSosal-Prism.zip', 'latest.mrpack', 'pack/pack.toml']
|
||||
(args.output / 'SHA256SUMS').write_text(''.join(f'{digest((args.output/f).read_bytes())} {f.replace("pack/", "current/")}\n' for f in files))
|
||||
print(json.dumps({'release':release, 'minecraft':deps['minecraft'], 'neoforge':deps['neoforge'], 'managed_files':len(entries), 'server':args.server}))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and publish the same tested server pack on both independent entry points."""
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('mrpack', type=Path)
|
||||
ap.add_argument('--bootstrap', type=Path)
|
||||
args = ap.parse_args()
|
||||
archive = args.mrpack.resolve()
|
||||
expected = hashlib.sha256(archive.read_bytes()).hexdigest()
|
||||
# Publishing a client before deploying its server can prevent players joining.
|
||||
deployed = subprocess.run([
|
||||
'ssh', '-o', 'BatchMode=yes', 'neuro',
|
||||
'systemctl is-active --quiet minecraft-server-wowMineMap && '
|
||||
'sha256sum /var/lib/minecraft-modpacks/worldOfSosal/WorldOfSosal.mrpack',
|
||||
], capture_output=True, text=True, check=True).stdout.split()[0]
|
||||
if deployed != expected:
|
||||
ap.error('Deploy this mrpack on neuro first: the server archive hash differs')
|
||||
scripts = Path(__file__).resolve().parent
|
||||
mirrors = [
|
||||
('hectic-lab', 'https://store.hectic-lab.com/minecraft/world-of-sosal/', 'store.hectic-lab.com:25568'),
|
||||
('bfs.poland.xray', 'https://store.bfs.band/minecraft/world-of-sosal/', 'wow.bfs.band'),
|
||||
]
|
||||
with tempfile.TemporaryDirectory(prefix='prism-mirrors-') as temporary:
|
||||
root = Path(temporary)
|
||||
bootstrap = args.bootstrap.resolve() if args.bootstrap else None
|
||||
builds = []
|
||||
for host, url, server in mirrors:
|
||||
output = root / host
|
||||
command = [sys.executable, str(scripts / 'build-prism-pack.py'),
|
||||
str(archive), str(output), '--base-url', url, '--server', server]
|
||||
if bootstrap:
|
||||
command += ['--bootstrap', str(bootstrap)]
|
||||
subprocess.run(command, check=True)
|
||||
if bootstrap is None:
|
||||
bootstrap = root / 'packwiz-installer-bootstrap.jar'
|
||||
with zipfile.ZipFile(output / 'WorldOfSosal-Prism.zip') as z:
|
||||
bootstrap.write_bytes(z.read('.minecraft/packwiz-installer-bootstrap.jar'))
|
||||
builds.append((host, url, output))
|
||||
# Each switch is atomic on its host. A failed publication exits nonzero;
|
||||
# rerunning safely verifies existing releases and retries both mirrors.
|
||||
for host, url, output in builds:
|
||||
subprocess.run([sys.executable, str(scripts / 'publish-prism-pack.py'),
|
||||
str(output), host], check=True)
|
||||
print(url, flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish build-prism-pack.py output; switch the complete pack atomically.
|
||||
|
||||
Usage: python3 script/publish-prism-pack.py OUTPUT_DIRECTORY [SSH_HOST]
|
||||
Coordinate gameplay updates with deployment of the same release on the server.
|
||||
"""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
root = Path(sys.argv[1]).resolve()
|
||||
host = sys.argv[2] if len(sys.argv) > 2 else 'hectic-lab'
|
||||
if host.startswith('-'):
|
||||
raise ValueError('Invalid SSH host')
|
||||
release = hashlib.sha256((root / 'pack/pack.toml').read_bytes()).hexdigest()
|
||||
with tempfile.TemporaryDirectory(prefix='prism-publish-') as temporary:
|
||||
archive = Path(temporary) / 'publish.tar.gz'
|
||||
with tarfile.open(archive, 'w:gz') as tar:
|
||||
for name in ['pack', 'WorldOfSosal-Prism.zip', 'latest.mrpack', 'SHA256SUMS', 'index.html']:
|
||||
tar.add(root / name, arcname=name)
|
||||
remote = f'/tmp/prism-publish-{release}.tar.gz'
|
||||
subprocess.run(['scp', '-o', 'BatchMode=yes', str(archive), f'{host}:{remote}'], check=True)
|
||||
script = '''set -eu
|
||||
umask 022
|
||||
base=/var/www/store/minecraft/world-of-sosal
|
||||
release=RELEASE
|
||||
archive=/tmp/prism-publish-$release.tar.gz
|
||||
mkdir -p "$base/releases"
|
||||
stage=$(mktemp -d "$base/.publish.XXXXXX")
|
||||
trap 'rm -rf "$stage"; rm -f "$archive"' EXIT
|
||||
tar -xzf "$archive" -C "$stage"
|
||||
chmod -R u=rwX,go=rX "$stage"
|
||||
if test -d "$base/releases/$release"; then
|
||||
diff -qr "$stage/pack" "$base/releases/$release"
|
||||
else
|
||||
mv "$stage/pack" "$base/releases/$release"
|
||||
fi
|
||||
for name in WorldOfSosal-Prism.zip latest.mrpack SHA256SUMS index.html; do
|
||||
mv "$stage/$name" "$base/$name"
|
||||
done
|
||||
ln -s "releases/$release" "$stage/current"
|
||||
mv -Tf "$stage/current" "$base/current"
|
||||
echo "Published $release"
|
||||
'''.replace('RELEASE', release)
|
||||
subprocess.run(['ssh', '-o', 'BatchMode=yes', host, 'sh', '-s'], input=script, text=True, check=True)
|
||||
print(f'Published client files on {host}:/var/www/store/minecraft/world-of-sosal')
|
||||
+4
-2
@@ -1,3 +1,5 @@
|
||||
immich:
|
||||
storage-box: ENC[AES256_GCM,data:eFNo4YaxJL25tM2icmwT+xrgkTybm20bKxCdc9JjG3ijmjM7FeOw/oQ4,iv:Y7GE+IfgQpz3Q4EvXWlPJnrFm08DkUT6qeDOr/nGFEM=,tag:3gtZA0v5dwGUlE34Ijz4Rg==,type:str]
|
||||
mailserver:
|
||||
security:
|
||||
hashedPassword: ENC[AES256_GCM,data:Z03x7tWHIhlRPaRZSrukyYOKhs6LdasZhZdizHdhlaJp2bywQZXKBaDABj2ab4rhwAPCHWhSiBjz35zV,iv:Z3hLC/A4YLVQkflr4cg9/wkKzo/RUdnLTwYC7ZhS0Hk=,tag:mSF/mbzH7iG6PwzyEsmyGg==,type:str]
|
||||
@@ -81,7 +83,7 @@ sops:
|
||||
Yk43ZmlTc09aNFV1VjdjN2RWQlFWTDQKcYSvA2lHP8GS0lkYY19Tm8RXmFHQX5Ck
|
||||
qV2Fn22Fic4M5FVKDEMfaO6WmeXgki9a8dGeO9LlC+Phf16SOq7eLw==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-06-06T12:32:43Z"
|
||||
mac: ENC[AES256_GCM,data:cmATu3qAaY7B74NvbejyMQMflh3akc0eLV+yal0NrTSE3rMd8rXYiAv1d2j/RxNd2ZLvnj5g6PFBP3O7v0JuYjKcNOvImz7SR2RRIaEWfT9Nl7XGtpTn5EpIi5KiL3unsqZTjvGrRywyLVfg0bxdKxX7RLnrBInX/zPVvMRNCL0=,iv:YpQdmsloPs3LrXA2FodziBCfxNfKrImPSe7TLNO1KH0=,tag:0By4hddfHl/0a5souDsJBw==,type:str]
|
||||
lastmodified: "2026-09-12T22:13:26Z"
|
||||
mac: ENC[AES256_GCM,data:NdCP9Vm8AQej0u3/6evlskRV4B5+QA3gNegQ5s5laoX+SwJNXnrJtAwZBKCWEZQKMb32qI+YVgnIJfn+HaJCkeaDqmc2I8d3DobiucraZypXh2VtWXGe4vCwzbhoc8KDb1eSNKt2e5CIzaR89BQYuwIHdngTSlzJ/qdc02qR1tA=,iv:VQcQoXpMlVCPZkREL35dwKSCCDTbAscDQk01mn9jht0=,tag:oaJTl+uj33/8WVoTv/j7Nw==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.10.2
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
minecraft:
|
||||
storage-box-key: ENC[AES256_GCM,data:Bf4hPWsFuS9u8DXeAYgvjQA7uCi7OHbq/C1yvhZsBmaKHBV7vfNCZoQrF0rE/Fo/ZAwfiirhxmn4FpNzakL8OzD99P0VPdC+BR/Fl2AA67l56UyTtHnIeir+BiWNlZuOOmhGCI1lrR8m0HW/HuY7BAK3wY9qyHDMyjti8fQBAE79axCnmpFR/cL3Fl+NV65IQwKVddhldTSW2kJhPAithjHHugObMJppn5jMPMGqu/MP9EsufnIc68F9X9tzFJEF4Ry7ukgTI3M8dtEX79jcssROXZAKl6oSiYG2CPmr8DGmbGyvkzy5m3MT9q/ZLrIZbNDPnWmDUVg+FFpqGlv4uLsWn/JIpuXev7t573MckjmKrVuuDldpArpVlKd3fgw2faxYD/iltH0cckfYSSqObndRWQcvN4ioJMUCXBIg/QlU111EcrqhwpaTkQIAXSZUxjUhlNEa+mfndxwp8TvV890zISwKRLXjv6AtrWy6m+9W2gm9ESbGRqDka4Ltzet0gcauY4Iwj6uzNukVfiZS,iv:33onrZNu3cMq5ukxuL8qZM2XeG1Qo+iaRFCy8Gy6U6k=,tag:jgKRJtp6mNZem7NrhDf1AA==,type:str]
|
||||
storage-box-pack-key: ENC[AES256_GCM,data:srm0+uu1ejWMdlxsIl4X6Fv2c/KBo4pVKcmtun+jqbsa1yifKBDT0hBul9dD/Vs6CojXZcDKTTs21B7zTfJuVmWX/z67qzY7aO0zNGtptgJsEW2OGwQjPWTXq6XhbIHGAGDutzluvdSpRCcKh9vKg2fPKHnhuntWGnDm/ujJKMgd5DqvapRNKLL4SoGo/6pedEVXJvEV9N37FvSoBIE2kj0KXxlX8AigjOtq7y+7Mk0p25FQY7uL1Ix94IK5Zi1VFt6gCbbntbPNeS08k+UUw+kDJ7QFwJrHIa7KzQ+3sL1eyKr5GbiWtDCzFGKSLth8qP8S2fgUQIII0L4Ak6NOSyJA3v1c1A8tl9kWiqZ/j0CQeFr++2PUi+lRI4tSo8stlV7MGA9lIO8Uwl0XTfdCkiEkBdZXj6UKyoDyS+hjTMGgh9evnCiEj0xETupA84CoRQEWXrb7wQmiw4evSJrYohiu+3iilNikgS9EJxCvXtoXemGTcqKCcXYvKRIPHmouLpeHYtLm2fpjKapvCGfV,iv:AkqaETKUQ7k6uH/nWV4KfxdGoBexH9r4ZiKDUZ/s4+0=,tag:3srUBlg2NvvFRq0BCoVvmg==,type:str]
|
||||
wow-tunnel-key: ENC[AES256_GCM,data:/2QeyAd+2+9VLeSEdZP5vh9sBqrfakxx0ZqNxzTr0QKT7d0s5jQpMMXOC763cjYFeEnvwQDpE4gL1lkoSSQ543vC0A0kA3a3AUUUtCQxTQqtrESnulferbp2jVXBXZrjYuIyH803YZ1iQD9rFaVEVQEtGR2lqkSqccMMvu+oSnJroXCJyuortN+xCri1xdPb+/tpqoffzpGet7TUUnAXAUatn9XaD18tj4q2ID9HfNLQdzwUjrFiEJx3yOFcn5PC1AE3REx6ZLIHF0ZDdVILfP/Q+uNa5l//Zrj9Xm2/+dgx7h7kGCwkS/Da8ow5EGpvcgAEsbX4cRgVLQvmnr2vaBbbArVyZpriMEyzEcwf68kl/KiZKgfEFuxPjXeawF/eLisMpwbqtFa1owljbzkBBTgBrCIvt9m29jiuj1EsNDdxvUsM95ATVr6eqDL04JgzPrt7niwzO5WeqBfugeOvNFlx3NCzKd7L0mG2qDPXTmRuOvRu3/LfctScW+zT9QCmN9IsxNkqFPLzhsF4JFD1HfPr3JLVsKYALJ5c,iv:k0y53lW8JLoQ8+Botf0EFzqSkTrvzaKB6fBbGTcdaLc=,tag:z4unb+jPbvKSlMZbORHYeA==,type:str]
|
||||
rcon-password: ENC[AES256_GCM,data:wzlMscGtdXF/Cssvi2qeNFgVtYVgCtm9nhX1rCVZ5KcnuyUmejjxsR6N/VAf4iTT,iv:BGdWotcNYNVL+Yi3Wd+5vYHw+l31Gt34MbTXsPxJEuM=,tag:VLWkYWLjjef8mvKYA0JgDA==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- recipient: age1r25zdeqq8nac6dgca9en28r57ffyz9u9d8z5yc25gc8xqz747vaqmdtk0h
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4NkhmaG5DeUIyZkZabHls
|
||||
bWM0Y2dmc2p0MkhtQnR4dUJWWjhkRzB6TnhNCnczdHpUenNtVEdVWit6UUFVQ0k5
|
||||
cTJhWnJ6Yi9mN0M2K1FiUHpqMjhwRm8KLS0tIFZCOTdybFpCY0huNUxtSVc3ZzQ5
|
||||
Ump2dXJyRS94elJYcGo0Y2svWCtkSGcKxnsHCGzvUYbgmEro9YkrvrOGJo7NYRRQ
|
||||
HzvXpMkoO64LHALsamQq0awmdpOPjXAnJ9UzgcXrLzgbyZPDyVSWng==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
- recipient: age1ak7heljpr0pjr4m0rcwxgn3sp0jjxw03lxyf33r8lcemqh2u2sgqx0aplq
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2ay9QbytUL3o1SjdPdTRV
|
||||
aVN0RGlTaWFpeU9Obkd2MkJoQWJFUy9rd2xzCkQ2N3NwYXZwSXdkWUNNbldNSElS
|
||||
MzZGdG83QWpsd2VBV1lLTXVwWk5tSUkKLS0tIFQvM3dzZVI2L050OXIxVEtFdVF4
|
||||
WWZseFVKV09RZmViQ1JEaDNFaUpTMkkKfUeGHjNCOfIeqxAxxNGUsFZ/yBCyzE8r
|
||||
+XeGgTZLRyfABCHi7uPkNYFMryMc4veGUdavTcaMNYUcNfEI/g3FtA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
- recipient: age1ev53mzse6rg4ffwtcwtq4e93c7x7s4d0eyu89jrsahrke8r4yamsseu8h4
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNY2FTRGZwQ2ZRSjVyYlpa
|
||||
WldKdmRKVS9JbHNScEJXbmVKbHRDZFU1NFZBClE5MitXSHc0ZHBHSnVPeHg3V0ZZ
|
||||
SFFXWWlwS2VMb0czSW93UllpZjhMSzgKLS0tIE5CL0FTOEhoNkhlMVdsS3pQbXNQ
|
||||
WWVLZDVCODJja2RqdTdZK3BmQzBXeDAK5Eqxf+HgCMhKLHtW0jRe6nDKHaSsGX/V
|
||||
nu5r7ceh5XHG1Mq3K9eA1oBa+GFP3305ch5ja6iTICj76ZXjmRnffg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-19T08:48:54Z"
|
||||
mac: ENC[AES256_GCM,data:vGdJ6avRiDu2PU/115cxCQNE5/Oj2K9QRLAxvPRp9UdYPrvx/sh2Q2cy5rGRL9pn09yZPB8LLtxFA9jDsZuSt8WIktIllkFfMb/ng3lEki/KSyd4YSlbA3wBcQ7wsWdfZ510BuRhbwCLnkIZpZYJ91mnaaU1ykmPzM5MHRj3IfQ=,iv:x/5jXo9ySW+3qqciS+bIDAwXo9O9j/I2JqcJp2BOLJQ=,tag:1yH4ZZReaP/dOuPyW0XBSw==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.10.2
|
||||
@@ -1,4 +1,9 @@
|
||||
{ system, inputs, self, pkgs }:
|
||||
let
|
||||
smtpPackages = map
|
||||
(version: self.packages.${system}."pg-${version}-ext-smtp-client")
|
||||
[ "15" "16" "17" ];
|
||||
in
|
||||
(import ./migrator { inherit system inputs self pkgs; }) //
|
||||
(import ./hemar { inherit system inputs self pkgs; }) //
|
||||
(import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) //
|
||||
@@ -7,4 +12,12 @@
|
||||
(import ./with-attic-cache { inherit system inputs self pkgs; }) //
|
||||
(import ./element-web { inherit system inputs self pkgs; }) //
|
||||
(import ./linux-devshell { inherit system inputs self pkgs; }) //
|
||||
(import ./windows-devshell { inherit system inputs self pkgs; })
|
||||
(import ./windows-devshell { inherit system inputs self pkgs; }) //
|
||||
{
|
||||
pg-smtp-client-metadata =
|
||||
assert builtins.all (package: package.pname == "pg_smtp_client") smtpPackages;
|
||||
assert builtins.all (package: package.version == "0.2.0") smtpPackages;
|
||||
pkgs.runCommand "pg-smtp-client-metadata" {} ''
|
||||
touch "$out"
|
||||
'';
|
||||
}
|
||||
|
||||
@@ -5,11 +5,36 @@ set -eu
|
||||
. "$STATE_SH"
|
||||
. "$DECIDE_SH"
|
||||
. "$HCLOUD_SH"
|
||||
. "$GITEA_SH"
|
||||
. "$CONTROLLER_SH"
|
||||
. "$WEBHOOK_SH"
|
||||
|
||||
gcr_state_init
|
||||
export GCR_ALLOWED_REPOS='hectic-lab/util.nix'
|
||||
export GCR_IMAGE_ID='313131'
|
||||
export GCR_NIX_IMAGE_ID='424242'
|
||||
|
||||
test "$(gcr_label_ttl ubuntu-latest)" = '180'
|
||||
test "$(gcr_label_ttl nix)" = '480'
|
||||
test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx23 180 0.004'
|
||||
test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx23 480 0.004'
|
||||
standard_candidates='cx23 nbg1 amd64
|
||||
cx23 fsn1 amd64
|
||||
cx23 hel1 amd64
|
||||
cx33 nbg1 amd64
|
||||
cx33 fsn1 amd64
|
||||
cx33 hel1 amd64
|
||||
cx43 nbg1 amd64
|
||||
cx43 fsn1 amd64
|
||||
cx43 hel1 amd64
|
||||
cx53 nbg1 amd64
|
||||
cx53 fsn1 amd64
|
||||
cx53 hel1 amd64'
|
||||
test "$(gcr_label_candidates ubuntu-latest)" = "$standard_candidates"
|
||||
test "$(gcr_label_candidates nix)" = "$standard_candidates"
|
||||
test "$(gcr_image_id_for_arch amd64 ubuntu-latest)" = '313131'
|
||||
test "$(gcr_image_id_for_arch amd64 nix)" = '424242'
|
||||
|
||||
profile="$(gcr_decide gross-nix-x86-highmem hectic-lab/util.nix)"
|
||||
test "$profile" = 'ccx53 480 0.8550'
|
||||
test "$(gcr_label_ttl gross-nix-x86-highmem)" = '480'
|
||||
@@ -30,33 +55,40 @@ if printf '%s\n' "$candidates" | grep -Evq '^ccx53 (nbg1|fsn1|hel1) amd64$'; the
|
||||
fi
|
||||
|
||||
export GCR_BUDGET_EUR_MONTHLY='6.83'
|
||||
if gcr_budget_add 0.8550 480; then
|
||||
if gcr_budget_can_add 0.8550 480; then
|
||||
printf 'highmem full-TTL reservation must obey budget cap\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
export GCR_BUDGET_EUR_MONTHLY='6.84'
|
||||
gcr_budget_add 0.8550 480
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '6.8400'
|
||||
if gcr_budget_write "$GCR_STATE_DIR/missing/budget" 1 2>/dev/null; then
|
||||
printf 'budget writes must propagate failures\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
|
||||
export GCR_BUDGET_EUR_MONTHLY='10'
|
||||
|
||||
calls="$GCR_STATE_DIR/hcloud-calls"
|
||||
sleep() { :; }
|
||||
gcr_hcloud_token() { printf token; }
|
||||
gcr_hcloud_req() {
|
||||
method="$1"; path="$2"; body="${3:-}"
|
||||
test "$method" = POST
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
if [ "$method" = GET ]; then
|
||||
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
|
||||
GCR_LAST_HTTP=200
|
||||
return 0
|
||||
fi
|
||||
test "$path" = /servers
|
||||
printf '%s\n' "$body" | jq -c . >> "$calls"
|
||||
count="$(wc -l < "$calls" | tr -d ' ')"
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
case "$count" in
|
||||
1|2) return 1 ;;
|
||||
3) printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
|
||||
1|2) GCR_LAST_HTTP=412; return 1 ;;
|
||||
3) GCR_LAST_HTTP=201; printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
vm_id="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
|
||||
test "$vm_id" = '9001'
|
||||
created="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
|
||||
test "$created" = '9001 ccx53 0.8550'
|
||||
test "$(wc -l < "$calls" | tr -d ' ')" = '3'
|
||||
jq -e 'select(.server_type == "ccx53" and .location == "nbg1" and .labels["gcr.arch"] == "amd64" and .labels["gcr.label"] == "gross-nix-x86-highmem" and .labels["gcr.ttl-min"] == "480")' "$calls" >/dev/null
|
||||
jq -e 'select(.server_type == "ccx53" and .location == "fsn1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null
|
||||
@@ -65,3 +97,210 @@ if jq -e 'select(.server_type != "ccx53" or .labels["gcr.arch"] != "amd64")' "$c
|
||||
printf 'highmem VM creation attempted non-ccx53 or non-amd64 candidate\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Standard allocation charges actual fallback type after cx23 capacity failures.
|
||||
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
|
||||
export GCR_BUDGET_EUR_MONTHLY='10'
|
||||
gcr_gitea_registration_token() { printf token; }
|
||||
gcr_hcloud_req() {
|
||||
method="$1"; path="$2"; body="${3:-}"
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
if [ "$method" = GET ]; then
|
||||
printf 'lookup\n' >> "$GCR_STATE_DIR/lookups"
|
||||
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
|
||||
GCR_LAST_HTTP=200
|
||||
return 0
|
||||
fi
|
||||
test "$path" = /servers
|
||||
printf '%s\n' "$body" | jq -c . >> "$calls"
|
||||
count="$(wc -l < "$calls" | tr -d ' ')"
|
||||
case "$count" in
|
||||
1|2|3) GCR_LAST_HTTP=412; return 1 ;;
|
||||
4) GCR_LAST_HTTP=201; printf '{"server":{"id":9002}}\n' > "$GCR_LAST_BODY"; return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
gcr_alloc 9002 1 hectic-lab/util.nix '["nix"]'
|
||||
test "$RESPONSE_CODE" = 202
|
||||
test "$RESPONSE_BODY" = 'allocated gcr-9002-1'
|
||||
test "$(wc -l < "$calls" | tr -d ' ')" = '4'
|
||||
test "$(wc -l < "$GCR_STATE_DIR/lookups" | tr -d ' ')" = '3'
|
||||
jq -e -s 'map(.server_type) == ["cx23", "cx23", "cx23", "cx33"]' "$calls" >/dev/null
|
||||
test "$(gcr_server_hourly_rate cx33)" = '0.008'
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0640'
|
||||
|
||||
# Ambiguous 5xx response adopts only exact deterministic managed identity.
|
||||
gcr_record_del 9002 1
|
||||
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
|
||||
gcr_hcloud_req() {
|
||||
method="$1"; body="${3:-}"
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
if [ "$method" = POST ]; then
|
||||
printf '%s\n' "$body" | jq -c . >> "$calls"
|
||||
GCR_LAST_HTTP=000
|
||||
return 1
|
||||
fi
|
||||
GCR_LAST_HTTP=200
|
||||
printf '%s\n' '{"servers":[{"id":9010,"name":"gcr-9010-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9010","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY"
|
||||
}
|
||||
gcr_alloc 9010 1 hectic-lab/util.nix '["nix"]'
|
||||
test "$RESPONSE_BODY" = 'allocated gcr-9010-1'
|
||||
test "$(wc -l < "$calls" | tr -d ' ')" = 1
|
||||
found_rec="$(gcr_record_get 9010 1)"
|
||||
test "$(gcr_record_field "$found_rec" status)" = pending_vm
|
||||
test "$(gcr_record_field "$found_rec" vm_id)" = 9010
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
|
||||
|
||||
# Ambiguous POST plus failed lookup retains reservation and blocks fallback;
|
||||
# reconciler adopts a later exact identity without deleting or refunding it.
|
||||
gcr_record_del 9010 1
|
||||
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" "$GCR_STATE_DIR/lookups"
|
||||
gcr_hcloud_req() {
|
||||
method="$1"; body="${3:-}"
|
||||
if [ "$method" = POST ]; then
|
||||
printf '%s\n' "$body" | jq -c . >> "$calls"
|
||||
GCR_LAST_HTTP=500
|
||||
return 1
|
||||
fi
|
||||
printf 'lookup-failed\n' >> "$GCR_STATE_DIR/lookups"
|
||||
GCR_LAST_HTTP=503
|
||||
return 1
|
||||
}
|
||||
gcr_alloc 9012 1 hectic-lab/util.nix '["nix"]'
|
||||
test "$RESPONSE_BODY" = 'VM creation pending recovery'
|
||||
test "$(wc -l < "$calls" | tr -d ' ')" = 1
|
||||
ambiguous_rec="$(gcr_record_get 9012 1)"
|
||||
test "$(gcr_record_field "$ambiguous_rec" status)" = create_ambiguous
|
||||
test "$(gcr_record_field "$ambiguous_rec" budget_rate)" = 0.004
|
||||
test "$(gcr_count_active)" = 1
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
|
||||
gcr_sweep_create_ambiguous
|
||||
test "$(gcr_record_field "$(gcr_record_get 9012 1)" status)" = create_ambiguous
|
||||
test "$(wc -l < "$calls" | tr -d ' ')" = 1
|
||||
|
||||
gcr_hcloud_req() {
|
||||
test "$1" = GET
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
GCR_LAST_HTTP=200
|
||||
printf '%s\n' '{"servers":[{"id":9012,"name":"gcr-9012-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9012","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY"
|
||||
}
|
||||
gcr_vm_destroy() { printf 'unexpected-destroy %s\n' "$1" >> "$GCR_STATE_DIR/ambiguity-destroys"; return 1; }
|
||||
gcr_sweep_create_ambiguous
|
||||
recovered_rec="$(gcr_record_get 9012 1)"
|
||||
test "$(gcr_record_field "$recovered_rec" status)" = pending_vm
|
||||
test "$(gcr_record_field "$recovered_rec" vm_id)" = 9012
|
||||
test "$(gcr_record_field "$recovered_rec" server_type)" = cx23
|
||||
test "$(gcr_record_field "$recovered_rec" budget_rate)" = 0.004
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
|
||||
test ! -e "$GCR_STATE_DIR/ambiguity-destroys"
|
||||
gcr_record_del 9012 1
|
||||
rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
|
||||
|
||||
# Fallback candidates exceeding remaining budget never reach Hetzner.
|
||||
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
|
||||
export GCR_BUDGET_EUR_MONTHLY='0.05'
|
||||
gcr_hcloud_req() {
|
||||
method="$1"; path="$2"; body="${3:-}"
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
if [ "$method" = GET ]; then
|
||||
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
|
||||
GCR_LAST_HTTP=200
|
||||
return 0
|
||||
fi
|
||||
test "$path" = /servers
|
||||
printf '%s\n' "$body" | jq -c . >> "$calls"
|
||||
GCR_LAST_HTTP=412
|
||||
return 1
|
||||
}
|
||||
gcr_alloc 9003 1 hectic-lab/util.nix '["nix"]'
|
||||
test "$RESPONSE_BODY" = 'VM creation failed'
|
||||
test "$(wc -l < "$calls" | tr -d ' ')" = '3'
|
||||
jq -e -s 'all(.server_type == "cx23")' "$calls" >/dev/null
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
|
||||
|
||||
# Deferred allocation keeps cleanup ownership and reservation across failed
|
||||
# DELETE, failed post-DELETE state update, and cleanup-owned 404 retry.
|
||||
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
|
||||
export GCR_BUDGET_EUR_MONTHLY='10'
|
||||
deferred='{"job_id":"9005","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":null,"vm_id":"","vm_name":"","status":"deferred"}'
|
||||
printf '%s\n' "$deferred" > "$GCR_STATE_DIR/jobs/9005-1.json"
|
||||
gcr_gitea_job_state() { printf queued; }
|
||||
record_put_count="$GCR_STATE_DIR/record-put-count"
|
||||
gcr_record_put() {
|
||||
count="$(cat "$record_put_count" 2>/dev/null || echo 0)"
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" > "$record_put_count"
|
||||
case "$count" in 1|3|5|8) return 1 ;; esac
|
||||
tmp="$(mktemp "$(dirname "$(gcr_record_path "$1" "$2")")/.tmp.XXXXXX")"
|
||||
printf '%s\n' "$3" > "$tmp"
|
||||
mv -f "$tmp" "$(gcr_record_path "$1" "$2")"
|
||||
}
|
||||
gcr_hcloud_req() {
|
||||
test "$1" = POST
|
||||
printf '%s\n' "$3" | jq -c . >> "$calls"
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
printf '{"server":{"id":9005}}\n' > "$GCR_LAST_BODY"
|
||||
}
|
||||
gcr_vm_destroy() {
|
||||
printf 'destroy-failed %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"
|
||||
return 1
|
||||
}
|
||||
gcr_alloc_deferred 9005 1
|
||||
cleanup_rec="$(gcr_record_get 9005 1)"
|
||||
test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
|
||||
test "$(gcr_record_field "$cleanup_rec" vm_id)" = 9005
|
||||
test "$(gcr_record_field "$cleanup_rec" budget_rate)" = 0.004
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
|
||||
|
||||
gcr_vm_destroy() { printf 'deleted %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"; }
|
||||
gcr_sweep_cleanup_pending
|
||||
cleanup_rec="$(gcr_record_get 9005 1)"
|
||||
test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
|
||||
test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = false
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
|
||||
gcr_vm_destroy() {
|
||||
printf 'absent %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"
|
||||
GCR_LAST_HTTP=404
|
||||
return 1
|
||||
}
|
||||
gcr_sweep_cleanup_pending
|
||||
cleanup_rec="$(gcr_record_get 9005 1)"
|
||||
test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = true
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
|
||||
gcr_sweep_cleanup_pending
|
||||
test ! -e "$(gcr_record_path 9005 1)"
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
|
||||
grep -q '^destroy-failed 9005$' "$GCR_STATE_DIR/destroy-calls"
|
||||
grep -q '^deleted 9005$' "$GCR_STATE_DIR/destroy-calls"
|
||||
grep -q '^absent 9005$' "$GCR_STATE_DIR/destroy-calls"
|
||||
test "$(grep -c '^absent 9005$' "$GCR_STATE_DIR/destroy-calls")" = 1
|
||||
|
||||
# Confirmed ambiguous absence uses same monotonic refund claim. Failed state
|
||||
# persistence after credit cannot make retry subtract reservation twice.
|
||||
gcr_budget_add 0.004 480
|
||||
ambiguous_absent='{"job_id":"9013","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":480,"vm_id":"","vm_name":"gcr-9013-1","server_type":"cx23","budget_rate":"0.004","candidate_location":"nbg1","candidate_arch":"amd64","bootstrapped":false,"status":"create_ambiguous"}'
|
||||
printf '%s\n' "$ambiguous_absent" > "$GCR_STATE_DIR/jobs/9013-1.json"
|
||||
gcr_hcloud_req() {
|
||||
test "$1" = GET
|
||||
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
|
||||
GCR_LAST_HTTP=200
|
||||
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
|
||||
}
|
||||
gcr_sweep_create_ambiguous
|
||||
absence_rec="$(gcr_record_get 9013 1)"
|
||||
test "$(gcr_record_field "$absence_rec" status)" = cleanup_pending
|
||||
test "$(gcr_record_field "$absence_rec" cleanup_vm_destroyed)" = true
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
|
||||
gcr_sweep_cleanup_pending
|
||||
test ! -e "$(gcr_record_path 9013 1)"
|
||||
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
|
||||
|
||||
# Reservation write failure aborts before any Hetzner request.
|
||||
rm -f "$calls"
|
||||
rm -rf "$GCR_STATE_DIR/budget"
|
||||
printf 'not-a-directory\n' > "$GCR_STATE_DIR/budget"
|
||||
gcr_alloc 9006 1 hectic-lab/util.nix '["nix"]'
|
||||
test "$RESPONSE_BODY" = 'VM creation failed'
|
||||
test ! -e "$calls"
|
||||
test ! -e "$(gcr_record_path 9006 1)"
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/bin/dash
|
||||
set -eu
|
||||
|
||||
. "$LOG_SH"
|
||||
. "$STATE_SH"
|
||||
. "$DECIDE_SH"
|
||||
. "$HCLOUD_SH"
|
||||
. "$GITEA_SH"
|
||||
. "$CONTROLLER_SH"
|
||||
. "$WEBHOOK_SH"
|
||||
|
||||
gcr_state_init
|
||||
export GCR_ALLOWED_REPOS='hinterland/hearth'
|
||||
export GCR_CONCURRENCY_CAP=2
|
||||
export GCR_PER_REPO_CAP=1
|
||||
NOW=2800
|
||||
gcr_now_epoch() { printf '%s' "$NOW"; }
|
||||
|
||||
calls="$GCR_STATE_DIR/calls"
|
||||
gcr_budget_add() { printf 'budget\n' >> "$calls"; }
|
||||
gcr_gitea_registration_token() { printf 'token'; printf 'token\n' >> "$calls"; }
|
||||
gcr_vm_create() {
|
||||
printf 'create\n' >> "$calls"
|
||||
gcr_budget_add 0.032 180
|
||||
printf '99 cx53 0.032'
|
||||
}
|
||||
gcr_vm_destroy() { printf 'destroy vm=%s\n' "$1" >> "$calls"; }
|
||||
gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"; }
|
||||
gcr_gitea_runner_disabled() { printf 'runner-disabled %s %s\n' "$2" "$3" >> "$calls"; }
|
||||
|
||||
original='{"job_id":"301","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":1000,"ttl_min":480,"vm_id":71,"vm_name":"gcr-301-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 301 1 "$original"
|
||||
gcr_deallocate 301 1 completed:success
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 4600)' \
|
||||
"$(gcr_record_path 301 1)" >/dev/null
|
||||
idle_once="$(gcr_record_get 301 1)"
|
||||
NOW=3000
|
||||
gcr_deallocate 301 1 completed:success
|
||||
test "$(gcr_record_get 301 1)" = "$idle_once"
|
||||
NOW=2800
|
||||
|
||||
# Exact elapsed hours keep current boundary instead of extending another hour.
|
||||
exact='{"job_id":"300","run_attempt":"1","repo":"hinterland/hearth","label":"gross-x86","created_at":1000,"ttl_min":180,"vm_id":70,"vm_name":"gcr-300-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 300 1 "$exact"
|
||||
NOW=4600
|
||||
gcr_deallocate 300 1 completed:success
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 4600)' \
|
||||
"$(gcr_record_path 300 1)" >/dev/null
|
||||
gcr_record_del 300 1
|
||||
NOW=2800
|
||||
gcr_deallocate 301 1 completed:success
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 4600)' \
|
||||
"$(gcr_record_path 301 1)" >/dev/null
|
||||
|
||||
# Exact billing boundary must expire now, not roll into another paid hour.
|
||||
NOW=4600
|
||||
boundary='{"job_id":"302","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":1000,"ttl_min":480,"vm_id":74,"vm_name":"gcr-302-1","bootstrapped":true,"status":"vm_active"}'
|
||||
test "$(gcr_record_idle_json "$boundary" | jq -r '.idle_expires_at')" = 4600
|
||||
NOW=2800
|
||||
|
||||
gcr_alloc 302 1 hinterland/hearth '["gross-nix-x86-perf"]'
|
||||
test "$RESPONSE_BODY" = 'reused gcr-301-1'
|
||||
test ! -e "$(gcr_record_path 301 1)"
|
||||
jq -e 'select(.job_id == "302" and .vm_id == 71 and
|
||||
.vm_name == "gcr-301-1" and .status == "pending_vm" and
|
||||
.created_at == 1000 and .assigned_at == 2800)' \
|
||||
"$(gcr_record_path 302 1)" >/dev/null
|
||||
test "$(grep -Ec '^(budget|token|create)$' "$calls" || true)" = 0
|
||||
|
||||
# Failed post-start health check keeps runner disabled and record retryable.
|
||||
retry_idle='{"job_id":"315","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":79,"vm_name":"gcr-315-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":4600}'
|
||||
gcr_record_put 315 1 "$retry_idle"
|
||||
export GCR_PER_REPO_CAP=2
|
||||
FAIL_HEALTH=1
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
[ "$2" = health ] && [ "$FAIL_HEALTH" = 1 ] && return 1
|
||||
return 0
|
||||
}
|
||||
gcr_alloc 316 1 hinterland/hearth '["gross-arm"]'
|
||||
retry_rec="$(gcr_record_get 316 1)"
|
||||
test "$(gcr_record_field "$retry_rec" bootstrapped)" = false
|
||||
test "$(gcr_record_field "$retry_rec" reused_vm)" = true
|
||||
grep -q '^runner start vm=79$' "$calls"
|
||||
grep -q '^runner health vm=79$' "$calls"
|
||||
grep -q '^runner-disabled gcr-315-1 true$' "$calls"
|
||||
if grep -q '^runner-disabled gcr-315-1 false$' "$calls"; then
|
||||
printf 'unhealthy reused runner must never become schedulable\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
FAIL_HEALTH=0
|
||||
gcr_record_del 316 1
|
||||
|
||||
# Expired idle capacity is never claimed; normal allocation then charges once.
|
||||
expired='{"job_id":"303","run_attempt":"1","repo":"hinterland/hearth","label":"gross-x86","created_at":1000,"ttl_min":180,"vm_id":72,"vm_name":"gcr-303-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":2800}'
|
||||
gcr_record_put 303 1 "$expired"
|
||||
NOW=2800
|
||||
export GCR_PER_REPO_CAP=2
|
||||
gcr_alloc 304 1 hinterland/hearth '["gross-x86"]'
|
||||
test "$RESPONSE_BODY" = 'allocated gcr-304-1'
|
||||
jq -e 'select(.vm_id == 99 and .status == "pending_vm")' \
|
||||
"$(gcr_record_path 304 1)" >/dev/null
|
||||
test "$(grep -c '^budget$' "$calls")" = 1
|
||||
test "$(grep -c '^token$' "$calls")" = 1
|
||||
test "$(grep -c '^create$' "$calls")" = 1
|
||||
|
||||
gcr_sweep_ttl
|
||||
grep -q 'destroy vm=72' "$calls"
|
||||
test ! -e "$(gcr_record_path 303 1)"
|
||||
|
||||
# Busy pool lock defers instead of racing into paid allocation.
|
||||
export GCR_CONCURRENCY_CAP=3
|
||||
export GCR_PER_REPO_CAP=3
|
||||
gcr_lock_acquire idle-pool
|
||||
gcr_alloc 305 1 hinterland/hearth '["gross-arm"]'
|
||||
test "$RESPONSE_BODY" = 'deferred: idle pool busy'
|
||||
test "$(gcr_record_field "$(gcr_record_get 305 1)" status)" = deferred
|
||||
test "$(grep -c '^create$' "$calls")" = 1
|
||||
gcr_lock_release idle-pool
|
||||
|
||||
# Claim needs at least one reconcile interval before slot and hard expiry.
|
||||
export GCR_RECONCILE_INTERVAL_SEC=60
|
||||
near_expiry='{"job_id":"313","run_attempt":"1","repo":"hinterland/hearth","label":"gross-mixed-econ","created_at":1000,"ttl_min":180,"vm_id":78,"vm_name":"gcr-313-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":2860}'
|
||||
gcr_record_put 313 1 "$near_expiry"
|
||||
NOW=2801
|
||||
gcr_lock_acquire "$(gcr_alloc_key 314 1)"
|
||||
if gcr_claim_idle 314 1 hinterland/hearth gross-mixed-econ; then
|
||||
printf 'near-expiry VM must not be reused\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
gcr_lock_release "$(gcr_alloc_key 314 1)"
|
||||
test -e "$(gcr_record_path 313 1)"
|
||||
test ! -e "$(gcr_record_path 314 1)"
|
||||
gcr_record_del 313 1
|
||||
NOW=2800
|
||||
|
||||
# Two concurrent claims transfer one VM once; original labels/name stay safe.
|
||||
race_idle='{"job_id":"306","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":73,"vm_name":"gcr-306-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":4600}'
|
||||
gcr_record_put 306 1 "$race_idle"
|
||||
claim_script="$GCR_STATE_DIR/claim.sh"
|
||||
cat > "$claim_script" <<'EOF'
|
||||
#!/bin/dash
|
||||
set -eu
|
||||
. "$STATE_SH"
|
||||
gcr_now_epoch() { printf '2800'; }
|
||||
key="$(gcr_alloc_key "$1" 1)"
|
||||
gcr_lock_acquire "$key"
|
||||
if gcr_claim_idle "$1" 1 hinterland/hearth gross-arm; then
|
||||
printf 'reused\n' > "$GCR_STATE_DIR/result-$1"
|
||||
else
|
||||
printf 'missed\n' > "$GCR_STATE_DIR/result-$1"
|
||||
fi
|
||||
gcr_lock_release "$key"
|
||||
EOF
|
||||
dash "$claim_script" 307 & first=$!
|
||||
dash "$claim_script" 308 & second=$!
|
||||
wait "$first"
|
||||
wait "$second"
|
||||
reused_count=0
|
||||
for result in "$GCR_STATE_DIR"/result-*; do
|
||||
[ "$(cat "$result")" = reused ] && reused_count=$((reused_count + 1))
|
||||
done
|
||||
test "$reused_count" = 1
|
||||
test ! -e "$(gcr_record_path 306 1)"
|
||||
|
||||
owner=""
|
||||
for job in 307 308; do
|
||||
rec="$(gcr_record_get "$job" 1)"
|
||||
if [ -n "$rec" ]; then
|
||||
test "$(gcr_record_field "$rec" vm_id)" = 73
|
||||
test "$(gcr_record_field "$rec" vm_name)" = gcr-306-1
|
||||
owner="$job"
|
||||
fi
|
||||
done
|
||||
test -n "$owner"
|
||||
|
||||
gcr_vm_list_managed() {
|
||||
printf 'list\n' >> "$GCR_STATE_DIR/list-calls"
|
||||
printf '[{"id":73,"labels":{"gcr.job-id":"306","gcr.run-attempt":"1"}}]'
|
||||
}
|
||||
gcr_gitea_list_runners() { printf '17 gcr-306-1\n'; }
|
||||
gcr_gitea_delete_runner() { printf 'delete repo=%s runner=%s\n' "$1" "$2" >> "$calls"; }
|
||||
before_destroy="$(grep -c '^destroy vm=73$' "$calls" || true)"
|
||||
gcr_lock_acquire admission
|
||||
gcr_sweep_orphan_vms
|
||||
test ! -e "$GCR_STATE_DIR/list-calls"
|
||||
gcr_lock_release admission
|
||||
gcr_sweep_orphan_vms
|
||||
test "$(wc -l < "$GCR_STATE_DIR/list-calls" | tr -d ' ')" = 1
|
||||
gcr_sweep_stale_runners
|
||||
after_destroy="$(grep -c '^destroy vm=73$' "$calls" || true)"
|
||||
test "$before_destroy" = "$after_destroy"
|
||||
if grep -q '^delete repo=.* runner=17$' "$calls"; then
|
||||
printf 'reused VM runner registration must not be swept as stale\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Interrupted destination-first transfer leaves duplicate state, never a VM
|
||||
# deletion: sweep drops superseded idle source and keeps active destination.
|
||||
duplicate_idle='{"job_id":"309","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":75,"vm_name":"gcr-309-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":2800}'
|
||||
duplicate_active='{"job_id":"310","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":75,"vm_name":"gcr-309-1","bootstrapped":true,"status":"pending_vm"}'
|
||||
gcr_record_put 309 1 "$duplicate_idle"
|
||||
gcr_record_put 310 1 "$duplicate_active"
|
||||
gcr_sweep_ttl
|
||||
test ! -e "$(gcr_record_path 309 1)"
|
||||
test -e "$(gcr_record_path 310 1)"
|
||||
if grep -q '^destroy vm=75$' "$calls"; then
|
||||
printf 'superseded idle record must not destroy reassigned VM\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Delayed old-job in_progress delivery cannot reactivate idle ownership.
|
||||
late_idle='{"job_id":"311","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":76,"vm_name":"gcr-311-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":4600}'
|
||||
gcr_record_put 311 1 "$late_idle"
|
||||
gcr_mark_in_progress 311 1
|
||||
test "$(gcr_record_field "$(gcr_record_get 311 1)" status)" = idle_vm
|
||||
|
||||
# Crash-stranded locks are reclaimed by dead owner PID; live contention remains.
|
||||
mkdir "$GCR_STATE_DIR/jobs/.lock.stale-test"
|
||||
if gcr_lock_acquire stale-test; then
|
||||
printf 'pre-existing lock must not be reclaimed\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -rf "$GCR_STATE_DIR/jobs/.lock.stale-test"
|
||||
mkdir "$GCR_STATE_DIR/jobs/.lock.crashed-test"
|
||||
printf '999999 1\n' > "$GCR_STATE_DIR/jobs/.lock.crashed-test/owner"
|
||||
gcr_lock_acquire crashed-test
|
||||
gcr_lock_release crashed-test
|
||||
|
||||
# Concurrent recovery of one dead lock admits exactly one owner.
|
||||
mkdir "$GCR_STATE_DIR/jobs/.lock.crashed-race"
|
||||
printf '999999 1\n' > "$GCR_STATE_DIR/jobs/.lock.crashed-race/owner"
|
||||
reclaim_script="$GCR_STATE_DIR/reclaim.sh"
|
||||
cat > "$reclaim_script" <<'EOF'
|
||||
#!/bin/dash
|
||||
set -eu
|
||||
. "$STATE_SH"
|
||||
if gcr_lock_acquire crashed-race; then
|
||||
printf 'acquired\n' > "$GCR_STATE_DIR/reclaim-$1"
|
||||
sleep 1
|
||||
gcr_lock_release crashed-race
|
||||
else
|
||||
printf 'busy\n' > "$GCR_STATE_DIR/reclaim-$1"
|
||||
fi
|
||||
EOF
|
||||
dash "$reclaim_script" first & first=$!
|
||||
dash "$reclaim_script" second & second=$!
|
||||
wait "$first"
|
||||
wait "$second"
|
||||
test "$(grep -lc '^acquired$' "$GCR_STATE_DIR"/reclaim-* | wc -l | tr -d ' ')" = 1
|
||||
mkdir "$GCR_STATE_DIR/jobs/.lock.crashed-test"
|
||||
printf '999999 1\n' > "$GCR_STATE_DIR/jobs/.lock.crashed-test/owner"
|
||||
gcr_lock_acquire crashed-test
|
||||
gcr_lock_release crashed-test
|
||||
|
||||
for job in 307 308 310 311; do
|
||||
gcr_record_del "$job" 1
|
||||
done
|
||||
ttl_record='{"job_id":"312","run_attempt":"1","repo":"hinterland/hearth","label":"gross-x86","created_at":1000,"ttl_min":180,"vm_id":77,"vm_name":"gcr-312-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 312 1 "$ttl_record"
|
||||
gcr_vm_public_ip() { printf '192.0.2.%s' "$1"; }
|
||||
gcr_vm_collect_diagnostics() {
|
||||
printf 'diag vm=%s job=%s reason=%s\n' "$1" "$3" "$4" >> "$calls"
|
||||
}
|
||||
NOW=11799
|
||||
gcr_sweep_ttl
|
||||
test -e "$(gcr_record_path 312 1)"
|
||||
NOW=11800
|
||||
gcr_sweep_ttl
|
||||
test ! -e "$(gcr_record_path 312 1)"
|
||||
grep -q '^diag vm=77 job=312 reason=ttl$' "$calls"
|
||||
grep -q '^destroy vm=77$' "$calls"
|
||||
|
||||
# Admission lock serializes cap check and creation across webhook processes.
|
||||
ADMISSION_STATE="$GCR_STATE_DIR/admission-state"
|
||||
mkdir "$ADMISSION_STATE"
|
||||
admission_script="$ADMISSION_STATE/allocate.sh"
|
||||
cat > "$admission_script" <<'EOF'
|
||||
#!/bin/dash
|
||||
set -eu
|
||||
. "$LOG_SH"
|
||||
. "$STATE_SH"
|
||||
. "$DECIDE_SH"
|
||||
. "$HCLOUD_SH"
|
||||
. "$WEBHOOK_SH"
|
||||
gcr_now_epoch() { printf '2000'; }
|
||||
gcr_budget_add() { printf 'budget %s\n' "$1" >> "$GCR_STATE_DIR/admission-calls"; }
|
||||
gcr_gitea_registration_token() { printf token; }
|
||||
gcr_vm_create() {
|
||||
sleep 1
|
||||
printf 'create %s\n' "$6" >> "$GCR_STATE_DIR/admission-calls"
|
||||
gcr_budget_add 0.032 180
|
||||
printf '%s cx53 0.032' "$6"
|
||||
}
|
||||
gcr_vm_destroy() { :; }
|
||||
gcr_state_init
|
||||
gcr_alloc "$1" 1 hinterland/hearth '["gross-x86"]'
|
||||
EOF
|
||||
old_state="$GCR_STATE_DIR"
|
||||
GCR_STATE_DIR="$ADMISSION_STATE" \
|
||||
GCR_ALLOWED_REPOS=hinterland/hearth \
|
||||
GCR_CONCURRENCY_CAP=1 \
|
||||
GCR_PER_REPO_CAP=1 \
|
||||
dash "$admission_script" 401 & first=$!
|
||||
GCR_STATE_DIR="$ADMISSION_STATE" \
|
||||
GCR_ALLOWED_REPOS=hinterland/hearth \
|
||||
GCR_CONCURRENCY_CAP=1 \
|
||||
GCR_PER_REPO_CAP=1 \
|
||||
dash "$admission_script" 402 & second=$!
|
||||
wait "$first"
|
||||
wait "$second"
|
||||
GCR_STATE_DIR="$ADMISSION_STATE"
|
||||
test "$(grep -c '^create ' "$GCR_STATE_DIR/admission-calls")" = 1
|
||||
active_count="$(gcr_count_active)"
|
||||
test "$active_count" = 1
|
||||
deferred_count="$(grep -El '"status"[[:space:]]*:[[:space:]]*"deferred"' \
|
||||
"$GCR_STATE_DIR"/jobs/*.json | wc -l | tr -d ' ')"
|
||||
test "$deferred_count" = 1
|
||||
GCR_STATE_DIR="$old_state"
|
||||
@@ -10,11 +10,13 @@ set -eu
|
||||
|
||||
gcr_state_init
|
||||
calls="$GCR_STATE_DIR/calls"
|
||||
gcr_now_epoch() { printf '1800'; }
|
||||
|
||||
gcr_gitea_job_state() {
|
||||
case "$2" in
|
||||
101) printf 'completed:success' ;;
|
||||
102) printf 'completed:failure' ;;
|
||||
105) printf 'completed:skipped' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -31,23 +33,87 @@ gcr_vm_destroy() {
|
||||
printf 'destroy vm=%s\n' "$1" >> "$calls"
|
||||
}
|
||||
|
||||
record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
}
|
||||
|
||||
gcr_gitea_runner_disabled() {
|
||||
printf 'disabled runner=%s value=%s\n' "$2" "$3" >> "$calls"
|
||||
}
|
||||
|
||||
record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"0","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}'
|
||||
record_failure='{"job_id":"102","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":42,"vm_name":"gcr-102-1","bootstrapped":true,"status":"vm_active"}'
|
||||
|
||||
gcr_record_put 101 1 "$record_success"
|
||||
gcr_record_put 102 1 "$record_failure"
|
||||
gcr_reap_finished_jobs
|
||||
|
||||
grep -q 'destroy vm=41' "$calls"
|
||||
grep -q 'destroy vm=42' "$calls"
|
||||
grep -q 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure' "$calls"
|
||||
grep -q 'runner health vm=42' "$calls"
|
||||
grep -q 'disabled runner=gcr-102-1 value=true' "$calls"
|
||||
grep -q 'runner stop vm=42' "$calls"
|
||||
test "$(grep -E '^(diag vm=42|runner health vm=42|disabled runner=gcr-102-1|runner stop vm=42)' "$calls")" = 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure
|
||||
runner health vm=42
|
||||
disabled runner=gcr-102-1 value=true
|
||||
runner stop vm=42'
|
||||
if grep -q 'diag vm=41' "$calls"; then
|
||||
printf 'success job should not collect diagnostics\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test ! -e "$(gcr_record_path 101 1)"
|
||||
test ! -e "$(gcr_record_path 102 1)"
|
||||
if grep -q 'destroy vm=41' "$calls"; then
|
||||
printf 'successful job VM should remain idle until billing boundary\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q 'destroy vm=42' "$calls"; then
|
||||
printf 'failed bootstrapped reconciled VM should remain idle until billing boundary\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3600)' \
|
||||
"$(gcr_record_path 101 1)" >/dev/null
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3601)' \
|
||||
"$(gcr_record_path 102 1)" >/dev/null
|
||||
gcr_idle_record_usable "$(gcr_record_get 102 1)"
|
||||
idle_once="$(gcr_record_get 101 1)"
|
||||
gcr_reap_finished_jobs
|
||||
test "$(gcr_record_get 101 1)" = "$idle_once"
|
||||
|
||||
# Skipped terminal jobs follow same healthy retention policy without diagnostics.
|
||||
record_skipped='{"job_id":"105","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":45,"vm_name":"gcr-105-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 105 1 "$record_skipped"
|
||||
gcr_reap_finished_jobs
|
||||
test "$(gcr_record_field "$(gcr_record_get 105 1)" status)" = idle_vm
|
||||
if grep -q 'diag vm=45' "$calls" || grep -q 'destroy vm=45' "$calls"; then
|
||||
printf 'healthy skipped VM must be retained without failure diagnostics\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Reaper keeps cleanup ownership after DELETE failure and retries next sweep.
|
||||
record_delete_fail='{"job_id":"104","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":44,"vm_name":"gcr-104-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 104 1 "$record_delete_fail"
|
||||
gcr_gitea_job_state() {
|
||||
case "$2" in
|
||||
104) printf 'completed:failure' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
gcr_vm_destroy() {
|
||||
printf 'destroy-failed vm=%s\n' "$1" >> "$calls"
|
||||
return 1
|
||||
}
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
[ "$1" != 44 ] || [ "$2" != stop ]
|
||||
}
|
||||
gcr_reap_finished_jobs
|
||||
test "$(gcr_record_field "$(gcr_record_get 104 1)" status)" = cleanup_pending
|
||||
test "$(gcr_count_active)" = 1
|
||||
|
||||
gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls"; }
|
||||
gcr_sweep_cleanup_pending
|
||||
test ! -e "$(gcr_record_path 104 1)"
|
||||
grep -q '^destroy-failed vm=44$' "$calls"
|
||||
grep -q '^destroy-retry vm=44$' "$calls"
|
||||
|
||||
calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
|
||||
calls="$calls_ip_fail"
|
||||
@@ -64,9 +130,18 @@ gcr_gitea_job_state() {
|
||||
gcr_vm_public_ip() {
|
||||
return 1
|
||||
}
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_vm_destroy() {
|
||||
printf 'destroy vm=%s\n' "$1" >> "$calls"
|
||||
}
|
||||
|
||||
gcr_reap_finished_jobs
|
||||
|
||||
grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail"
|
||||
grep -q 'runner health vm=43' "$calls_ip_fail"
|
||||
grep -q 'destroy vm=43' "$calls_ip_fail"
|
||||
test ! -e "$(gcr_record_path 103 1)"
|
||||
|
||||
@@ -10,6 +10,7 @@ set -eu
|
||||
|
||||
gcr_state_init
|
||||
calls="$GCR_STATE_DIR/calls"
|
||||
gcr_now_epoch() { printf '1800'; }
|
||||
|
||||
gcr_vm_public_ip() {
|
||||
printf '192.0.2.%s' "$1"
|
||||
@@ -23,7 +24,15 @@ gcr_vm_destroy() {
|
||||
printf 'destroy vm=%s\n' "$1" >> "$calls"
|
||||
}
|
||||
|
||||
record_success='{"job_id":"201","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":51,"vm_name":"gcr-201-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
}
|
||||
|
||||
gcr_gitea_runner_disabled() {
|
||||
printf 'disabled runner=%s value=%s\n' "$2" "$3" >> "$calls"
|
||||
}
|
||||
|
||||
record_success='{"job_id":"201","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"0","ttl_min":480,"vm_id":51,"vm_name":"gcr-201-1","bootstrapped":true,"status":"vm_active"}'
|
||||
record_failure='{"job_id":"202","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":52,"vm_name":"gcr-202-1","bootstrapped":true,"status":"vm_active"}'
|
||||
|
||||
gcr_record_put 201 1 "$record_success"
|
||||
@@ -31,16 +40,53 @@ gcr_record_put 202 1 "$record_failure"
|
||||
gcr_deallocate 201 1 completed:success
|
||||
gcr_deallocate 202 1 completed:failure
|
||||
|
||||
grep -q 'destroy vm=51' "$calls"
|
||||
grep -q 'destroy vm=52' "$calls"
|
||||
grep -q 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure' "$calls"
|
||||
grep -q 'runner health vm=52' "$calls"
|
||||
grep -q 'disabled runner=gcr-202-1 value=true' "$calls"
|
||||
grep -q 'runner stop vm=52' "$calls"
|
||||
test "$(grep -E '^(diag vm=52|runner health vm=52|disabled runner=gcr-202-1|runner stop vm=52)' "$calls")" = 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure
|
||||
runner health vm=52
|
||||
disabled runner=gcr-202-1 value=true
|
||||
runner stop vm=52'
|
||||
if grep -q 'diag vm=51' "$calls"; then
|
||||
printf 'success webhook should not collect diagnostics\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test ! -e "$(gcr_record_path 201 1)"
|
||||
test ! -e "$(gcr_record_path 202 1)"
|
||||
if grep -q 'destroy vm=51' "$calls"; then
|
||||
printf 'successful webhook VM should remain idle until billing boundary\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q 'destroy vm=52' "$calls"; then
|
||||
printf 'failed bootstrapped webhook VM should remain idle until billing boundary\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3600)' \
|
||||
"$(gcr_record_path 201 1)" >/dev/null
|
||||
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3601)' \
|
||||
"$(gcr_record_path 202 1)" >/dev/null
|
||||
gcr_idle_record_usable "$(gcr_record_get 202 1)"
|
||||
idle_once="$(gcr_record_get 201 1)"
|
||||
gcr_record_del 201 1
|
||||
gcr_lock_acquire "$(gcr_alloc_key 205 1)"
|
||||
gcr_claim_idle 205 1 hinterland/hearth gross-nix-x86-perf
|
||||
gcr_lock_release "$(gcr_alloc_key 205 1)"
|
||||
jq -e 'select(.job_id == "205" and .vm_id == 52 and .status == "pending_vm" and .reused_vm == true)' \
|
||||
"$(gcr_record_path 205 1)" >/dev/null
|
||||
gcr_record_del 205 1
|
||||
gcr_record_put 201 1 "$idle_once"
|
||||
gcr_deallocate 201 1 completed:success
|
||||
test "$(gcr_record_get 201 1)" = "$idle_once"
|
||||
|
||||
# Cancelled terminal jobs use same retention policy without failure diagnostics.
|
||||
record_cancelled='{"job_id":"207","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":57,"vm_name":"gcr-207-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 207 1 "$record_cancelled"
|
||||
gcr_deallocate 207 1 completed:cancelled
|
||||
test "$(gcr_record_field "$(gcr_record_get 207 1)" status)" = idle_vm
|
||||
if grep -q 'diag vm=57' "$calls" || grep -q 'destroy vm=57' "$calls"; then
|
||||
printf 'healthy cancelled VM must be retained without failure diagnostics\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
|
||||
calls="$calls_ip_fail"
|
||||
@@ -50,9 +96,77 @@ gcr_record_put 203 1 "$record_ip_fail"
|
||||
gcr_vm_public_ip() {
|
||||
return 1
|
||||
}
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
return 1
|
||||
}
|
||||
|
||||
gcr_deallocate 203 1 completed:failure
|
||||
|
||||
grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail"
|
||||
grep -q 'runner health vm=53' "$calls_ip_fail"
|
||||
grep -q 'destroy vm=53' "$calls_ip_fail"
|
||||
test ! -e "$(gcr_record_path 203 1)"
|
||||
if grep -q 'disabled runner=gcr-203-1' "$calls_ip_fail"; then
|
||||
printf 'unhealthy terminal runner must not enter idle shutdown path\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Runner teardown failure destroys and retains ownership until DELETE succeeds.
|
||||
record_delete_fail='{"job_id":"204","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":54,"vm_name":"gcr-204-1","bootstrapped":true,"status":"vm_active"}'
|
||||
gcr_record_put 204 1 "$record_delete_fail"
|
||||
gcr_vm_destroy() {
|
||||
printf 'destroy-failed vm=%s\n' "$1" >> "$calls_ip_fail"
|
||||
return 1
|
||||
}
|
||||
gcr_vm_runner_service() {
|
||||
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
|
||||
[ "$2" != stop ]
|
||||
}
|
||||
gcr_deallocate 204 1 completed:failure
|
||||
cleanup_rec="$(gcr_record_get 204 1)"
|
||||
test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
|
||||
test "$(gcr_count_active)" = 1
|
||||
test "$(gcr_count_active_repo hinterland/hearth)" = 1
|
||||
|
||||
gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls_ip_fail"; }
|
||||
key="$(gcr_alloc_key 204 1)"
|
||||
gcr_lock_acquire "$key"
|
||||
gcr_vm_cleanup_pending 204 1 "$(gcr_record_get 204 1)"
|
||||
gcr_lock_release "$key"
|
||||
test ! -e "$(gcr_record_path 204 1)"
|
||||
test "$(gcr_count_active)" = 0
|
||||
grep -q '^destroy-failed vm=54$' "$calls_ip_fail"
|
||||
grep -q '^destroy-retry vm=54$' "$calls_ip_fail"
|
||||
|
||||
# Unbootstrapped terminal VMs are never retained.
|
||||
gcr_vm_runner_service() { printf 'unexpected-runner %s\n' "$1" >> "$calls_ip_fail"; }
|
||||
gcr_vm_destroy() { printf 'destroy-unbootstrapped vm=%s\n' "$1" >> "$calls_ip_fail"; }
|
||||
record_unbootstrapped='{"job_id":"206","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":56,"vm_name":"gcr-206-1","bootstrapped":false,"status":"pending_vm"}'
|
||||
gcr_record_put 206 1 "$record_unbootstrapped"
|
||||
gcr_deallocate 206 1 completed:failure
|
||||
grep -q '^destroy-unbootstrapped vm=56$' "$calls_ip_fail"
|
||||
test ! -e "$(gcr_record_path 206 1)"
|
||||
if grep -q '^unexpected-runner 56$' "$calls_ip_fail"; then
|
||||
printf 'unbootstrapped VM must bypass idle teardown\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Gitea emits zero-based run_attempt values for initial workflow jobs.
|
||||
gcr_read_request() {
|
||||
gcr_hdr_event_type=workflow_job
|
||||
gcr_hdr_delivery=test-delivery
|
||||
gcr_hdr_signature=test-signature
|
||||
gcr_body='{"action":"queued","workflow_job":{"id":331,"run_attempt":0,"labels":["nix"]},"repository":{"full_name":"hinterland/hearth"}}'
|
||||
}
|
||||
gcr_verify_signature() { :; }
|
||||
gcr_alloc() {
|
||||
printf '%s:%s:%s:%s\n' "$1" "$2" "$3" "$4" > "$GCR_STATE_DIR/allocation"
|
||||
RESPONSE_CODE=202
|
||||
RESPONSE_BODY=allocated
|
||||
}
|
||||
gcr_respond() { printf '%s:%s\n' "$1" "$2" > "$GCR_STATE_DIR/response"; }
|
||||
|
||||
gcr_handle_webhook
|
||||
test "$(cat "$GCR_STATE_DIR/allocation")" = '331:0:hinterland/hearth:["nix"]'
|
||||
test "$(cat "$GCR_STATE_DIR/response")" = '202:allocated'
|
||||
|
||||
Reference in New Issue
Block a user