fix: world-of-sosal

docs/project-zomboid-backups.md

@@ -3,22 +3,24 @@
 `hectic.services."project-zomboid".backup` creates local backups without stopping
 or pausing the server. The default schedule is every 30 minutes. Each run:
 
-1. rsyncs `Zomboid/Saves/Multiplayer/<serverName>` and non-secret server
+1. sends the local RCON `save` command and waits for the configured save grace
+   period;
+2. rsyncs `Zomboid/Saves/Multiplayer/<serverName>` and non-secret server
    settings (`SandboxVars`, spawn-points, and spawn-regions) from
    `Zomboid/Server` into a private staging tree;
-2. waits five seconds and repeats the rsync to narrow the live-write window;
+3. waits five seconds and repeats the rsync to narrow the live-write window;
-3. publishes a timestamped `tar.zst` archive; and
+4. publishes a timestamped `tar.zst` archive; and
-4. deletes local archives older than `backup.retentionDays`.
+5. deletes local archives older than `backup.retentionDays`.
 
 The service lock prevents overlapping runs. Missing save or server-config paths
 skip the run through systemd `ConditionPathExists` checks.
 
 ## Consistency and secrets
 
-This is a best-effort, crash-consistent backup. It does not stop Project
+This is a best-effort backup. It does not stop Project Zomboid and does not use
-Zomboid and does not use an atomic filesystem snapshot. A backup taken during a
+an atomic filesystem snapshot. The RCON save command flushes the world before
-busy save can therefore contain files from slightly different moments; the
+copying, and the second rsync narrows the remaining live-write window, but
-second rsync reduces but cannot remove this risk.
+neither makes the filesystem copy an atomic snapshot.
 
 Archives do not include the generated server INI, `admin-password`,
 host-generated password files, or the S3 credentials file. The server INI is
@@ -41,12 +43,17 @@ systemctl status project-zomboid-backup.service
 journalctl -u project-zomboid-backup.service
 ```
 
+RCON is enabled on localhost port `27015`; the firewall does not expose this
+port. The password is generated at
+`/var/lib/project-zomboid/rcon-password` with mode `0600`. The server also uses
+`SaveWorldEveryMinutes=15` as a periodic persistence fallback.
+
 ## Optional S3 upload
 
 S3 upload is disabled by default. Enabling it requires `bucket`, `endpoint`,
 `region`, and an absolute runtime `credentialsFile` outside `/nix/store`. The
 endpoint must use HTTPS. systemd reads the environment file without executing
-it; keep it root-owned and mode `0400`:
+it; this host keeps it owned by `project-zomboid` with mode `0400`:
 
 ```sh
 AWS_ACCESS_KEY_ID=...
@@ -64,6 +71,18 @@ available; it remains the stronger recovery and cleanup control.
 Restoring must be done while the server is stopped so it cannot modify files
 during extraction:
 
+The versioned helper creates a fresh current-state backup, stops the timer and
+server, validates archive paths, restores the save, and starts both services:
+
+```sh
+sudo ./docs/project-zomboid-restore.sh \
+  /var/lib/project-zomboid/backups/archive/<archive>.tar.zst
+```
+
+It writes a rollback archive named
+`project-zomboid-<serverName>-pre-restore-<timestamp>.tar.zst` before changing
+the save.
+
 ```sh
 systemctl stop project-zomboid.service
 tar --zstd --no-same-owner --no-same-permissions \

docs/project-zomboid-restore.sh

@@ -0,0 +1,143 @@
+#!/bin/sh
+set -eu
+
+SERVER_NAME=${SERVER_NAME:-servertest}
+DATA_DIR=${DATA_DIR:-/var/lib/project-zomboid}
+ARCHIVE=${1:-}
+
+usage() {
+  printf '%s\n' "Usage: $0 /path/to/project-zomboid-${SERVER_NAME}-<timestamp>.tar.zst"
+  printf '%s\n' "Environment: SERVER_NAME, DATA_DIR"
+}
+
+if [ "$(id -u)" -ne 0 ]; then
+  printf '%s\n' 'Run as root.' >&2
+  exit 1
+fi
+
+if [ -z "$ARCHIVE" ]; then
+  usage >&2
+  exit 2
+fi
+
+if [ ! -r "$ARCHIVE" ]; then
+  printf 'Backup archive is not readable: %s\n' "$ARCHIVE" >&2
+  exit 1
+fi
+
+ARCHIVE_DIR="$DATA_DIR/backups/archive"
+SAVE_DIR="$DATA_DIR/Zomboid/Saves/Multiplayer/$SERVER_NAME"
+SERVER_DIR="$DATA_DIR/Zomboid/Server"
+TMP_LIST=$(mktemp)
+ROLLBACK_ARCHIVE=''
+SERVER_STOPPED=0
+RESTORE_SUCCEEDED=0
+
+cleanup() {
+  rm -f "$TMP_LIST"
+}
+
+on_exit() {
+  status=$?
+  if [ "$status" -ne 0 ] && [ "$SERVER_STOPPED" -eq 1 ] \
+    && [ "$RESTORE_SUCCEEDED" -eq 0 ] && [ -n "$ROLLBACK_ARCHIVE" ]; then
+    set +e
+    rm -rf "$SAVE_DIR"
+    rm -f \
+      "$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \
+      "$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \
+      "$SERVER_DIR/${SERVER_NAME}_spawnregions.lua"
+    tar --zstd --no-same-owner --no-same-permissions \
+      -xpf "$ROLLBACK_ARCHIVE" -C "$DATA_DIR"
+    chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR"
+    systemctl start project-zomboid.service
+    systemctl start project-zomboid-backup.timer
+    printf '%s\n' "Restore failed; current state restored from $ROLLBACK_ARCHIVE" >&2
+  fi
+  cleanup
+  exit "$status"
+}
+trap on_exit EXIT
+
+if ! tar --zstd -tf "$ARCHIVE" >"$TMP_LIST"; then
+  printf 'Archive integrity check failed: %s\n' "$ARCHIVE" >&2
+  exit 1
+fi
+
+while IFS= read -r member; do
+  case "$member" in
+    Zomboid/*) ;;
+    *)
+      printf 'Unsafe archive member: %s\n' "$member" >&2
+      exit 1
+      ;;
+  esac
+  case "$member" in
+    /*|*../*)
+      printf 'Path traversal member: %s\n' "$member" >&2
+      exit 1
+      ;;
+  esac
+done <"$TMP_LIST"
+
+if ! systemctl start project-zomboid-backup.service; then
+  printf '%s\n' 'Could not create fresh backup of current state.' >&2
+  exit 1
+fi
+
+CURRENT_ARCHIVE=$(find "$ARCHIVE_DIR" -maxdepth 1 -type f \
+  -name "project-zomboid-$SERVER_NAME-*.tar.zst" \
+  -printf '%T@ %p\n' | sort -nr | awk 'NR == 1 {sub(/^[^ ]* /, ""); print}')
+
+if [ -z "$CURRENT_ARCHIVE" ]; then
+  printf '%s\n' 'Fresh current-state backup was not found.' >&2
+  exit 1
+fi
+
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+ROLLBACK_ARCHIVE="$ARCHIVE_DIR/project-zomboid-$SERVER_NAME-pre-restore-$stamp.tar.zst"
+cp --reflink=auto "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE" 2>/dev/null \
+  || cp "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE"
+chmod 0600 "$ROLLBACK_ARCHIVE"
+chown project-zomboid:project-zomboid "$ROLLBACK_ARCHIVE"
+
+systemctl stop project-zomboid-backup.timer
+systemctl stop project-zomboid.service
+SERVER_STOPPED=1
+
+if [ "$(systemctl show project-zomboid --property=ActiveState --value)" != inactive ]; then
+  printf '%s\n' 'Project Zomboid did not stop; refusing to restore.' >&2
+  exit 1
+fi
+
+rm -rf "$SAVE_DIR"
+rm -f \
+  "$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \
+  "$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \
+  "$SERVER_DIR/${SERVER_NAME}_spawnregions.lua"
+
+tar --zstd --no-same-owner --no-same-permissions \
+  -xpf "$ARCHIVE" -C "$DATA_DIR"
+chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR"
+
+systemctl start project-zomboid.service
+started=0
+for _ in $(seq 1 90); do
+  if [ "$(systemctl show project-zomboid --property=ActiveState --value)" = active ] \
+    && [ "$(systemctl show project-zomboid --property=SubState --value)" = running ]; then
+    started=1
+    break
+  fi
+  sleep 2
+done
+
+if [ "$started" -ne 1 ]; then
+  printf 'Restore completed, but service did not become healthy. Rollback: %s\n' \
+    "$ROLLBACK_ARCHIVE" >&2
+  exit 1
+fi
+
+systemctl start project-zomboid-backup.timer
+RESTORE_SUCCEEDED=1
+printf 'Restore completed.\n'
+printf 'Rollback archive: %s\n' "$ROLLBACK_ARCHIVE"

nixos/module/hectic/service/project-zomboid.nix

@@ -37,6 +37,7 @@
   ) cfg.sandboxProperties;
   zomboidDir = "${cfg.dataDir}/Zomboid";
   adminPasswordFile = "${cfg.dataDir}/admin-password";
+  rconPasswordFile = cfg.rcon.passwordFile;
   backupCfg = cfg.backup;
   s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile;
   s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
@@ -62,6 +63,20 @@
       exit 0
     fi
 
+    ${lib.optionalString cfg.rcon.enable ''
+      rcon_password="$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})"
+      if [ -z "$rcon_password" ]; then
+        ${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid RCON password file is empty.' >&2
+        exit 1
+      fi
+      ${pkgs.rcon}/bin/rcon \
+        --host 127.0.0.1 \
+        --port ${toString cfg.rcon.port} \
+        --password "$rcon_password" \
+        save
+      ${pkgs.coreutils}/bin/sleep ${toString backupCfg.saveWaitSeconds}
+    ''}
+
     sync_staging() {
       ${pkgs.rsync}/bin/rsync -a --delete \
         ${lib.escapeShellArg "${saveDir}/"} \
@@ -273,6 +288,22 @@ in {
       description = "Open the Project Zomboid UDP ports in the firewall.";
     };
 
+    rcon = {
+      enable = lib.mkEnableOption "local RCON for Project Zomboid automation";
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 27015;
+        description = "RCON TCP port; not opened in the firewall by this module.";
+      };
+
+      passwordFile = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/rcon-password";
+        description = "Runtime file containing the generated RCON password.";
+      };
+    };
+
     backup = {
       enable = lib.mkEnableOption "no-stop Project Zomboid backups";
 
@@ -300,6 +331,12 @@ in {
         description = "Delete local archives older than this many days.";
       };
 
+      saveWaitSeconds = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 10;
+        description = "Seconds to wait after the RCON save command before rsync.";
+      };
+
       s3 = {
         enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage";
 
@@ -347,6 +384,13 @@ in {
 
   config = lib.mkIf cfg.enable {
     assertions = [
+      {
+        assertion = !cfg.rcon.enable || (
+          lib.hasPrefix "/" cfg.rcon.passwordFile
+          && !lib.hasPrefix "/nix/store/" cfg.rcon.passwordFile
+        );
+        message = "hectic.services.project-zomboid.rcon.passwordFile must be a runtime path outside /nix/store.";
+      }
       {
         assertion = !backupCfg.s3.enable || backupCfg.enable;
         message = "hectic.services.project-zomboid.backup must be enabled before S3 upload.";
@@ -417,6 +461,22 @@ in {
           umask 077
           ${pkgs.openssl}/bin/openssl rand -base64 32 > ${lib.escapeShellArg adminPasswordFile}
         fi
+        ${lib.optionalString cfg.rcon.enable ''
+          if [ ! -s ${lib.escapeShellArg rconPasswordFile} ]; then
+            umask 077
+            ${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile}
+          else
+            rcon_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})
+            case "$rcon_password" in
+              *[!0123456789abcdefABCDEF]*)
+                umask 077
+                ${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile}
+                ;;
+            esac
+          fi
+          ${pkgs.coreutils}/bin/chown project-zomboid:project-zomboid ${lib.escapeShellArg rconPasswordFile}
+          ${pkgs.coreutils}/bin/chmod 0600 ${lib.escapeShellArg rconPasswordFile}
+        ''}
         ${pkgs.steamcmd}/bin/steamcmd \
           +force_install_dir ${lib.escapeShellArg cfg.installDir} \
           +login anonymous \
@@ -433,6 +493,12 @@ in {
           ) configLines}
           ${lib.optionalString (cfg.serverPropertiesFile != null)
             "${pkgs.coreutils}/bin/cat ${lib.escapeShellArg cfg.serverPropertiesFile};"}
+          ${lib.optionalString cfg.rcon.enable ''
+            ${pkgs.coreutils}/bin/printf '%s\n' ${lib.escapeShellArg "RCONPort=${toString cfg.rcon.port}"};
+            ${pkgs.coreutils}/bin/printf '%s' 'RCONPassword=';
+            ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile};
+            ${pkgs.coreutils}/bin/printf '\n';
+          ''}
         } > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}.ini"}
         ${lib.optionalString (cfg.sandboxProperties != { }) ''
           {

nixos/system/hectic-lab/hectic-lab.nix

@@ -106,14 +106,22 @@ in {
       memory = "3g";
       serverName = "servertest";
       serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
+      rcon.enable = true;
       backup = {
         enable = true;
         onCalendar = "*:0/30";
         retentionDays = 14;
-        s3.enable = false;
+        s3 = {
+          enable = true;
+          bucket = "backup-hectic-lab";
+          endpoint = "https://hel1.your-objectstorage.com";
+          region = "hel1";
+          credentialsFile = "/var/lib/project-zomboid/s3-credentials";
+        };
       };
       serverProperties = {
         Map = "Muldraugh, KY";
+        SaveWorldEveryMinutes = 15;
         DoLuaChecksum = false;
         Public = true;
         AntiCheatSafety = 4;
@@ -259,12 +267,37 @@ in {
       "jwt-secret"
       "s3-access-key"
       "s3-secret-key"
-    ]);
+    ]) // {
+      "project-zomboid/s3-access-key" = {
+        key = "ente/s3-access-key";
+        owner = "project-zomboid";
+        group = "project-zomboid";
+      };
+      "project-zomboid/s3-secret-key" = {
+        key = "ente/s3-secret-key";
+        owner = "project-zomboid";
+        group = "project-zomboid";
+      };
+    };
   };
 
   systemd.services.project-zomboid.preStart = lib.mkBefore ''
     password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"}
     properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"}
+    s3_credentials_file=${lib.escapeShellArg "/var/lib/project-zomboid/s3-credentials"}
+    s3_credentials_tmp="$(${pkgs.coreutils}/bin/mktemp "''${s3_credentials_file}.XXXXXX")"
+    trap '${pkgs.coreutils}/bin/rm -f "$s3_credentials_tmp"' EXIT
+
+    {
+      ${pkgs.coreutils}/bin/printf 'AWS_ACCESS_KEY_ID='
+      ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-access-key".path}
+      ${pkgs.coreutils}/bin/printf '\n'
+      ${pkgs.coreutils}/bin/printf 'AWS_SECRET_ACCESS_KEY='
+      ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-secret-key".path}
+      ${pkgs.coreutils}/bin/printf '\n'
+    } > "$s3_credentials_tmp"
+    ${pkgs.coreutils}/bin/chmod 0400 "$s3_credentials_tmp"
+    ${pkgs.coreutils}/bin/mv -f "$s3_credentials_tmp" "$s3_credentials_file"
 
     if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then
       umask 077

nixos/system/neuro/minecraft/world-of-sosal.nix

@@ -13,10 +13,10 @@
     serverName = "wowMineMap";
     remoteHost = "u664722.your-storagebox.de";
     remoteUser = "u664722";
-    remotePath = "minecraft/pack/WorldOfSosal.mrpack";
+    remotePath = "minecraft/pack/WorldOfSosal-v3.mrpack";
     archiveName = "WorldOfSosal.mrpack";
     cacheDir = "/var/lib/minecraft-modpacks/worldOfSosal";
-    archiveSha256 = "f8c18acb9208e4592725632ae50dab4f9c308483b34fd43a6507c74fdbf8169f";
+    archiveSha256 = "f97cf251b14f40590e97e7b39e8a8ec43dacfce6da1b02357d15e0eee10d3ade";
     expectedDependencies = {
       minecraft = "1.21.1";
       neoforge = "21.1.250";