From e444ea5936cbe43f884bd4afa94e5a97b399a0a9 Mon Sep 17 00:00:00 2001 From: yukkop Date: Wed, 23 Sep 2026 19:26:19 +0000 Subject: [PATCH] fix: world-of-sosal --- docs/project-zomboid-backups.md | 37 +++-- docs/project-zomboid-restore.sh | 143 ++++++++++++++++++ .../module/hectic/service/project-zomboid.nix | 66 ++++++++ nixos/system/hectic-lab/hectic-lab.nix | 37 ++++- .../system/neuro/minecraft/world-of-sosal.nix | 4 +- 5 files changed, 274 insertions(+), 13 deletions(-) create mode 100755 docs/project-zomboid-restore.sh diff --git a/docs/project-zomboid-backups.md b/docs/project-zomboid-backups.md index 7436ab7d..9ec187af 100644 --- a/docs/project-zomboid-backups.md +++ b/docs/project-zomboid-backups.md @@ -3,22 +3,24 @@ `hectic.services."project-zomboid".backup` creates local backups without stopping or pausing the server. The default schedule is every 30 minutes. Each run: -1. rsyncs `Zomboid/Saves/Multiplayer/` and non-secret server +1. sends the local RCON `save` command and waits for the configured save grace + period; +2. rsyncs `Zomboid/Saves/Multiplayer/` and non-secret server settings (`SandboxVars`, spawn-points, and spawn-regions) from `Zomboid/Server` into a private staging tree; -2. waits five seconds and repeats the rsync to narrow the live-write window; -3. publishes a timestamped `tar.zst` archive; and -4. deletes local archives older than `backup.retentionDays`. +3. waits five seconds and repeats the rsync to narrow the live-write window; +4. publishes a timestamped `tar.zst` archive; and +5. deletes local archives older than `backup.retentionDays`. The service lock prevents overlapping runs. Missing save or server-config paths skip the run through systemd `ConditionPathExists` checks. ## Consistency and secrets -This is a best-effort, crash-consistent backup. It does not stop Project -Zomboid and does not use an atomic filesystem snapshot. A backup taken during a -busy save can therefore contain files from slightly different moments; the -second rsync reduces but cannot remove this risk. +This is a best-effort backup. It does not stop Project Zomboid and does not use +an atomic filesystem snapshot. The RCON save command flushes the world before +copying, and the second rsync narrows the remaining live-write window, but +neither makes the filesystem copy an atomic snapshot. Archives do not include the generated server INI, `admin-password`, host-generated password files, or the S3 credentials file. The server INI is @@ -41,12 +43,17 @@ systemctl status project-zomboid-backup.service journalctl -u project-zomboid-backup.service ``` +RCON is enabled on localhost port `27015`; the firewall does not expose this +port. The password is generated at +`/var/lib/project-zomboid/rcon-password` with mode `0600`. The server also uses +`SaveWorldEveryMinutes=15` as a periodic persistence fallback. + ## Optional S3 upload S3 upload is disabled by default. Enabling it requires `bucket`, `endpoint`, `region`, and an absolute runtime `credentialsFile` outside `/nix/store`. The endpoint must use HTTPS. systemd reads the environment file without executing -it; keep it root-owned and mode `0400`: +it; this host keeps it owned by `project-zomboid` with mode `0400`: ```sh AWS_ACCESS_KEY_ID=... @@ -64,6 +71,18 @@ available; it remains the stronger recovery and cleanup control. Restoring must be done while the server is stopped so it cannot modify files during extraction: +The versioned helper creates a fresh current-state backup, stops the timer and +server, validates archive paths, restores the save, and starts both services: + +```sh +sudo ./docs/project-zomboid-restore.sh \ + /var/lib/project-zomboid/backups/archive/.tar.zst +``` + +It writes a rollback archive named +`project-zomboid--pre-restore-.tar.zst` before changing +the save. + ```sh systemctl stop project-zomboid.service tar --zstd --no-same-owner --no-same-permissions \ diff --git a/docs/project-zomboid-restore.sh b/docs/project-zomboid-restore.sh new file mode 100755 index 00000000..251b13ed --- /dev/null +++ b/docs/project-zomboid-restore.sh @@ -0,0 +1,143 @@ +#!/bin/sh +set -eu + +SERVER_NAME=${SERVER_NAME:-servertest} +DATA_DIR=${DATA_DIR:-/var/lib/project-zomboid} +ARCHIVE=${1:-} + +usage() { + printf '%s\n' "Usage: $0 /path/to/project-zomboid-${SERVER_NAME}-.tar.zst" + printf '%s\n' "Environment: SERVER_NAME, DATA_DIR" +} + +if [ "$(id -u)" -ne 0 ]; then + printf '%s\n' 'Run as root.' >&2 + exit 1 +fi + +if [ -z "$ARCHIVE" ]; then + usage >&2 + exit 2 +fi + +if [ ! -r "$ARCHIVE" ]; then + printf 'Backup archive is not readable: %s\n' "$ARCHIVE" >&2 + exit 1 +fi + +ARCHIVE_DIR="$DATA_DIR/backups/archive" +SAVE_DIR="$DATA_DIR/Zomboid/Saves/Multiplayer/$SERVER_NAME" +SERVER_DIR="$DATA_DIR/Zomboid/Server" +TMP_LIST=$(mktemp) +ROLLBACK_ARCHIVE='' +SERVER_STOPPED=0 +RESTORE_SUCCEEDED=0 + +cleanup() { + rm -f "$TMP_LIST" +} + +on_exit() { + status=$? + if [ "$status" -ne 0 ] && [ "$SERVER_STOPPED" -eq 1 ] \ + && [ "$RESTORE_SUCCEEDED" -eq 0 ] && [ -n "$ROLLBACK_ARCHIVE" ]; then + set +e + rm -rf "$SAVE_DIR" + rm -f \ + "$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \ + "$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \ + "$SERVER_DIR/${SERVER_NAME}_spawnregions.lua" + tar --zstd --no-same-owner --no-same-permissions \ + -xpf "$ROLLBACK_ARCHIVE" -C "$DATA_DIR" + chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR" + systemctl start project-zomboid.service + systemctl start project-zomboid-backup.timer + printf '%s\n' "Restore failed; current state restored from $ROLLBACK_ARCHIVE" >&2 + fi + cleanup + exit "$status" +} +trap on_exit EXIT + +if ! tar --zstd -tf "$ARCHIVE" >"$TMP_LIST"; then + printf 'Archive integrity check failed: %s\n' "$ARCHIVE" >&2 + exit 1 +fi + +while IFS= read -r member; do + case "$member" in + Zomboid/*) ;; + *) + printf 'Unsafe archive member: %s\n' "$member" >&2 + exit 1 + ;; + esac + case "$member" in + /*|*../*) + printf 'Path traversal member: %s\n' "$member" >&2 + exit 1 + ;; + esac +done <"$TMP_LIST" + +if ! systemctl start project-zomboid-backup.service; then + printf '%s\n' 'Could not create fresh backup of current state.' >&2 + exit 1 +fi + +CURRENT_ARCHIVE=$(find "$ARCHIVE_DIR" -maxdepth 1 -type f \ + -name "project-zomboid-$SERVER_NAME-*.tar.zst" \ + -printf '%T@ %p\n' | sort -nr | awk 'NR == 1 {sub(/^[^ ]* /, ""); print}') + +if [ -z "$CURRENT_ARCHIVE" ]; then + printf '%s\n' 'Fresh current-state backup was not found.' >&2 + exit 1 +fi + +stamp=$(date -u +%Y%m%dT%H%M%SZ) +ROLLBACK_ARCHIVE="$ARCHIVE_DIR/project-zomboid-$SERVER_NAME-pre-restore-$stamp.tar.zst" +cp --reflink=auto "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE" 2>/dev/null \ + || cp "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE" +chmod 0600 "$ROLLBACK_ARCHIVE" +chown project-zomboid:project-zomboid "$ROLLBACK_ARCHIVE" + +systemctl stop project-zomboid-backup.timer +systemctl stop project-zomboid.service +SERVER_STOPPED=1 + +if [ "$(systemctl show project-zomboid --property=ActiveState --value)" != inactive ]; then + printf '%s\n' 'Project Zomboid did not stop; refusing to restore.' >&2 + exit 1 +fi + +rm -rf "$SAVE_DIR" +rm -f \ + "$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \ + "$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \ + "$SERVER_DIR/${SERVER_NAME}_spawnregions.lua" + +tar --zstd --no-same-owner --no-same-permissions \ + -xpf "$ARCHIVE" -C "$DATA_DIR" +chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR" + +systemctl start project-zomboid.service +started=0 +for _ in $(seq 1 90); do + if [ "$(systemctl show project-zomboid --property=ActiveState --value)" = active ] \ + && [ "$(systemctl show project-zomboid --property=SubState --value)" = running ]; then + started=1 + break + fi + sleep 2 +done + +if [ "$started" -ne 1 ]; then + printf 'Restore completed, but service did not become healthy. Rollback: %s\n' \ + "$ROLLBACK_ARCHIVE" >&2 + exit 1 +fi + +systemctl start project-zomboid-backup.timer +RESTORE_SUCCEEDED=1 +printf 'Restore completed.\n' +printf 'Rollback archive: %s\n' "$ROLLBACK_ARCHIVE" diff --git a/nixos/module/hectic/service/project-zomboid.nix b/nixos/module/hectic/service/project-zomboid.nix index 24172c2c..84e9c7e9 100644 --- a/nixos/module/hectic/service/project-zomboid.nix +++ b/nixos/module/hectic/service/project-zomboid.nix @@ -37,6 +37,7 @@ ) cfg.sandboxProperties; zomboidDir = "${cfg.dataDir}/Zomboid"; adminPasswordFile = "${cfg.dataDir}/admin-password"; + rconPasswordFile = cfg.rcon.passwordFile; backupCfg = cfg.backup; s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile; s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket; @@ -62,6 +63,20 @@ exit 0 fi + ${lib.optionalString cfg.rcon.enable '' + rcon_password="$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})" + if [ -z "$rcon_password" ]; then + ${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid RCON password file is empty.' >&2 + exit 1 + fi + ${pkgs.rcon}/bin/rcon \ + --host 127.0.0.1 \ + --port ${toString cfg.rcon.port} \ + --password "$rcon_password" \ + save + ${pkgs.coreutils}/bin/sleep ${toString backupCfg.saveWaitSeconds} + ''} + sync_staging() { ${pkgs.rsync}/bin/rsync -a --delete \ ${lib.escapeShellArg "${saveDir}/"} \ @@ -273,6 +288,22 @@ in { description = "Open the Project Zomboid UDP ports in the firewall."; }; + rcon = { + enable = lib.mkEnableOption "local RCON for Project Zomboid automation"; + + port = lib.mkOption { + type = lib.types.port; + default = 27015; + description = "RCON TCP port; not opened in the firewall by this module."; + }; + + passwordFile = lib.mkOption { + type = lib.types.path; + default = "${cfg.dataDir}/rcon-password"; + description = "Runtime file containing the generated RCON password."; + }; + }; + backup = { enable = lib.mkEnableOption "no-stop Project Zomboid backups"; @@ -300,6 +331,12 @@ in { description = "Delete local archives older than this many days."; }; + saveWaitSeconds = lib.mkOption { + type = lib.types.ints.positive; + default = 10; + description = "Seconds to wait after the RCON save command before rsync."; + }; + s3 = { enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage"; @@ -347,6 +384,13 @@ in { config = lib.mkIf cfg.enable { assertions = [ + { + assertion = !cfg.rcon.enable || ( + lib.hasPrefix "/" cfg.rcon.passwordFile + && !lib.hasPrefix "/nix/store/" cfg.rcon.passwordFile + ); + message = "hectic.services.project-zomboid.rcon.passwordFile must be a runtime path outside /nix/store."; + } { assertion = !backupCfg.s3.enable || backupCfg.enable; message = "hectic.services.project-zomboid.backup must be enabled before S3 upload."; @@ -417,6 +461,22 @@ in { umask 077 ${pkgs.openssl}/bin/openssl rand -base64 32 > ${lib.escapeShellArg adminPasswordFile} fi + ${lib.optionalString cfg.rcon.enable '' + if [ ! -s ${lib.escapeShellArg rconPasswordFile} ]; then + umask 077 + ${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile} + else + rcon_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile}) + case "$rcon_password" in + *[!0123456789abcdefABCDEF]*) + umask 077 + ${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile} + ;; + esac + fi + ${pkgs.coreutils}/bin/chown project-zomboid:project-zomboid ${lib.escapeShellArg rconPasswordFile} + ${pkgs.coreutils}/bin/chmod 0600 ${lib.escapeShellArg rconPasswordFile} + ''} ${pkgs.steamcmd}/bin/steamcmd \ +force_install_dir ${lib.escapeShellArg cfg.installDir} \ +login anonymous \ @@ -433,6 +493,12 @@ in { ) configLines} ${lib.optionalString (cfg.serverPropertiesFile != null) "${pkgs.coreutils}/bin/cat ${lib.escapeShellArg cfg.serverPropertiesFile};"} + ${lib.optionalString cfg.rcon.enable '' + ${pkgs.coreutils}/bin/printf '%s\n' ${lib.escapeShellArg "RCONPort=${toString cfg.rcon.port}"}; + ${pkgs.coreutils}/bin/printf '%s' 'RCONPassword='; + ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile}; + ${pkgs.coreutils}/bin/printf '\n'; + ''} } > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}.ini"} ${lib.optionalString (cfg.sandboxProperties != { }) '' { diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index 8af83a3e..ec7373ca 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -106,14 +106,22 @@ in { memory = "3g"; serverName = "servertest"; serverPropertiesFile = /var/lib/project-zomboid/server-password.ini; + rcon.enable = true; backup = { enable = true; onCalendar = "*:0/30"; retentionDays = 14; - s3.enable = false; + s3 = { + enable = true; + bucket = "backup-hectic-lab"; + endpoint = "https://hel1.your-objectstorage.com"; + region = "hel1"; + credentialsFile = "/var/lib/project-zomboid/s3-credentials"; + }; }; serverProperties = { Map = "Muldraugh, KY"; + SaveWorldEveryMinutes = 15; DoLuaChecksum = false; Public = true; AntiCheatSafety = 4; @@ -259,12 +267,37 @@ in { "jwt-secret" "s3-access-key" "s3-secret-key" - ]); + ]) // { + "project-zomboid/s3-access-key" = { + key = "ente/s3-access-key"; + owner = "project-zomboid"; + group = "project-zomboid"; + }; + "project-zomboid/s3-secret-key" = { + key = "ente/s3-secret-key"; + owner = "project-zomboid"; + group = "project-zomboid"; + }; + }; }; systemd.services.project-zomboid.preStart = lib.mkBefore '' password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"} properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"} + s3_credentials_file=${lib.escapeShellArg "/var/lib/project-zomboid/s3-credentials"} + s3_credentials_tmp="$(${pkgs.coreutils}/bin/mktemp "''${s3_credentials_file}.XXXXXX")" + trap '${pkgs.coreutils}/bin/rm -f "$s3_credentials_tmp"' EXIT + + { + ${pkgs.coreutils}/bin/printf 'AWS_ACCESS_KEY_ID=' + ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-access-key".path} + ${pkgs.coreutils}/bin/printf '\n' + ${pkgs.coreutils}/bin/printf 'AWS_SECRET_ACCESS_KEY=' + ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-secret-key".path} + ${pkgs.coreutils}/bin/printf '\n' + } > "$s3_credentials_tmp" + ${pkgs.coreutils}/bin/chmod 0400 "$s3_credentials_tmp" + ${pkgs.coreutils}/bin/mv -f "$s3_credentials_tmp" "$s3_credentials_file" if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then umask 077 diff --git a/nixos/system/neuro/minecraft/world-of-sosal.nix b/nixos/system/neuro/minecraft/world-of-sosal.nix index 6c1b70ff..2cead090 100644 --- a/nixos/system/neuro/minecraft/world-of-sosal.nix +++ b/nixos/system/neuro/minecraft/world-of-sosal.nix @@ -13,10 +13,10 @@ serverName = "wowMineMap"; remoteHost = "u664722.your-storagebox.de"; remoteUser = "u664722"; - remotePath = "minecraft/pack/WorldOfSosal.mrpack"; + remotePath = "minecraft/pack/WorldOfSosal-v3.mrpack"; archiveName = "WorldOfSosal.mrpack"; cacheDir = "/var/lib/minecraft-modpacks/worldOfSosal"; - archiveSha256 = "f8c18acb9208e4592725632ae50dab4f9c308483b34fd43a6507c74fdbf8169f"; + archiveSha256 = "f97cf251b14f40590e97e7b39e8a8ec43dacfce6da1b02357d15e0eee10d3ade"; expectedDependencies = { minecraft = "1.21.1"; neoforge = "21.1.250";