feat: zomboid backups

nixos/module/hectic/service/project-zomboid.nix

@@ -37,6 +37,147 @@
   ) cfg.sandboxProperties;
   zomboidDir = "${cfg.dataDir}/Zomboid";
   adminPasswordFile = "${cfg.dataDir}/admin-password";
+  backupCfg = cfg.backup;
+  s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile;
+  s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
+  s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint;
+  s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region;
+  saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}";
+  serverConfigDir = "${zomboidDir}/Server";
+  backupScript = pkgs.writeShellScript "project-zomboid-backup" ''
+    set -eu
+
+    staging_dir=${lib.escapeShellArg backupCfg.stagingDir}
+    archive_dir=${lib.escapeShellArg backupCfg.archiveDir}
+    lock_file="$archive_dir/.backup.lock"
+
+    ${pkgs.coreutils}/bin/install -d -m 0700 \
+      "$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}" \
+      "$staging_dir/Zomboid/Server" \
+      "$archive_dir"
+
+    exec 9>"$lock_file"
+    if ! ${pkgs.util-linux}/bin/flock -n 9; then
+      ${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid backup already running; skipping.' >&2
+      exit 0
+    fi
+
+    sync_staging() {
+      ${pkgs.rsync}/bin/rsync -a --delete \
+        ${lib.escapeShellArg "${saveDir}/"} \
+        "$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}/"
+      ${pkgs.rsync}/bin/rsync -a --delete --delete-excluded \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_SandboxVars.lua"} \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_spawnpoints.lua"} \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_spawnregions.lua"} \
+        --exclude='*' \
+        ${lib.escapeShellArg "${serverConfigDir}/"} \
+        "$staging_dir/Zomboid/Server/"
+    }
+
+    # Second pass narrows, but cannot eliminate, live-save inconsistency.
+    sync_staging
+    ${pkgs.coreutils}/bin/sleep 5
+    sync_staging
+
+    timestamp="$(${pkgs.coreutils}/bin/date -u +%Y%m%dT%H%M%SZ)"
+    archive_name="project-zomboid-${cfg.serverName}-$timestamp.tar.zst"
+    archive_tmp="$archive_dir/.$archive_name.tmp"
+    archive="$archive_dir/$archive_name"
+    trap '${pkgs.coreutils}/bin/rm -f "$archive_tmp"' EXIT
+    ${pkgs.gnutar}/bin/tar \
+      --use-compress-program=${lib.escapeShellArg "${pkgs.zstd}/bin/zstd -T0"} \
+      -C "$staging_dir" -cf "$archive_tmp" Zomboid
+    ${pkgs.coreutils}/bin/chmod 0600 "$archive_tmp"
+    ${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive"
+    trap - EXIT
+
+    ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+      -name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \
+      -mmin +${toString (backupCfg.retentionDays * 1440)} -delete
+
+    ${lib.optionalString backupCfg.s3.enable ''
+      if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
+        ${pkgs.coreutils}/bin/printf '%s\n' \
+          'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
+        exit 1
+      fi
+      s3_bucket=${lib.escapeShellArg s3Bucket}
+      s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
+      s3_key="''${s3_prefix:+$s3_prefix/}$archive_name"
+      ${pkgs.awscli2}/bin/aws s3 cp "$archive" \
+        "s3://$s3_bucket/$s3_key" \
+        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+        --region ${lib.escapeShellArg s3Region} \
+        --cli-connect-timeout 30 \
+        --cli-read-timeout 300 \
+        --only-show-errors
+
+      remote_prefix="$s3_prefix"
+      if [ -n "$remote_prefix" ]; then
+        remote_prefix="$remote_prefix/"
+      fi
+      archive_prefix=${lib.escapeShellArg "project-zomboid-${cfg.serverName}-"}
+      remote_list="$staging_dir/.remote-objects.json"
+      remote_delete_dir="$staging_dir/.remote-delete"
+      ${pkgs.awscli2}/bin/aws s3api list-objects-v2 \
+        --bucket "$s3_bucket" \
+        --prefix "$remote_prefix" \
+        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+        --region ${lib.escapeShellArg s3Region} \
+        --output json > "$remote_list"
+      ${pkgs.python3}/bin/python3 - "$remote_list" "$remote_delete_dir" \
+        "$(( $(${pkgs.coreutils}/bin/date +%s) - ${toString (backupCfg.s3.remoteRetentionDays * 86400)} ))" \
+        "$remote_prefix$archive_prefix" <<'PY'
+import datetime
+import json
+import os
+import re
+import sys
+
+objects_path, delete_dir, cutoff, key_prefix = sys.argv[1:]
+cutoff = int(cutoff)
+archive_pattern = re.compile(
+    re.escape(key_prefix) + r"\d{8}T\d{6}Z\.tar\.zst\Z"
+)
+with open(objects_path, encoding="utf-8") as stream:
+    objects = json.load(stream).get("Contents", [])
+
+old_keys = []
+for item in objects:
+    key = item.get("Key", "")
+    if not archive_pattern.fullmatch(key):
+        continue
+    modified = datetime.datetime.fromisoformat(
+        item["LastModified"].replace("Z", "+00:00")
+    )
+    if int(modified.timestamp()) < cutoff:
+        old_keys.append(key)
+
+os.makedirs(delete_dir, exist_ok=True)
+for batch_number in range(0, len(old_keys), 1000):
+    batch = old_keys[batch_number:batch_number + 1000]
+    manifest_path = os.path.join(
+        delete_dir, f"batch-{batch_number // 1000:04d}.json"
+    )
+    with open(manifest_path, "w", encoding="utf-8") as stream:
+        json.dump(
+            {"Objects": [{"Key": key} for key in batch], "Quiet": True},
+            stream,
+        )
+PY
+      for remote_manifest in "$remote_delete_dir"/*.json; do
+        [ -f "$remote_manifest" ] || continue
+        ${pkgs.awscli2}/bin/aws s3api delete-objects \
+          --bucket "$s3_bucket" \
+          --delete "file://$remote_manifest" \
+          --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+          --region ${lib.escapeShellArg s3Region} \
+          --only-show-errors
+      done
+      ${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir"
+    ''}
+  '';
   startScript = pkgs.writeShellScript "project-zomboid-start" ''
     admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile})
     exec ${pkgs.steam-run}/bin/steam-run \
@@ -131,9 +272,120 @@ in {
       default = true;
       description = "Open the Project Zomboid UDP ports in the firewall.";
     };
+
+    backup = {
+      enable = lib.mkEnableOption "no-stop Project Zomboid backups";
+
+      onCalendar = lib.mkOption {
+        type = lib.types.str;
+        default = "*:0/30";
+        description = "systemd calendar expression controlling backup frequency.";
+      };
+
+      stagingDir = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/backups/staging";
+        description = "Local directory containing the two-pass rsync staging tree.";
+      };
+
+      archiveDir = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/backups/archive";
+        description = "Local directory containing timestamped tar.zst archives.";
+      };
+
+      retentionDays = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 14;
+        description = "Delete local archives older than this many days.";
+      };
+
+      s3 = {
+        enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage";
+
+        credentialsFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            Runtime env file containing AWS_ACCESS_KEY_ID and
+            AWS_SECRET_ACCESS_KEY. Required when S3 upload is enabled.
+          '';
+        };
+
+        bucket = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3 bucket receiving backup archives.";
+        };
+
+        endpoint = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3-compatible endpoint URL.";
+        };
+
+        region = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3 region passed to awscli2.";
+        };
+
+        prefix = lib.mkOption {
+          type = lib.types.str;
+          default = "project-zomboid";
+          description = "Optional object key prefix within the S3 bucket.";
+        };
+
+        remoteRetentionDays = lib.mkOption {
+          type = lib.types.ints.positive;
+          default = 14;
+          description = "Delete uploaded archives older than this many days.";
+        };
+      };
+    };
   };
 
   config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.enable;
+        message = "hectic.services.project-zomboid.backup must be enabled before S3 upload.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.credentialsFile != null;
+        message = "hectic.services.project-zomboid.backup.s3.credentialsFile is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.bucket != null;
+        message = "hectic.services.project-zomboid.backup.s3.bucket is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.endpoint != null;
+        message = "hectic.services.project-zomboid.backup.s3.endpoint is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.region != null;
+        message = "hectic.services.project-zomboid.backup.s3.region is required when S3 upload is enabled.";
+      }
+      {
+        assertion =
+          !backupCfg.s3.enable
+          || backupCfg.s3.credentialsFile == null
+          || (
+            lib.hasPrefix "/" backupCfg.s3.credentialsFile
+            && !lib.hasPrefix "/nix/store/" backupCfg.s3.credentialsFile
+          );
+        message = "hectic.services.project-zomboid.backup.s3.credentialsFile must be a runtime path outside /nix/store.";
+      }
+      {
+        assertion =
+          !backupCfg.s3.enable
+          || backupCfg.s3.endpoint == null
+          || lib.hasPrefix "https://" backupCfg.s3.endpoint;
+        message = "hectic.services.project-zomboid.backup.s3.endpoint must use HTTPS.";
+      }
+    ];
+
     users.groups.project-zomboid = { };
     users.users.project-zomboid = {
       isSystemUser = true;
@@ -145,6 +397,11 @@ in {
     systemd.tmpfiles.rules = [
       "d ${cfg.dataDir} 0750 project-zomboid project-zomboid - -"
       "d ${cfg.installDir} 0750 project-zomboid project-zomboid - -"
+    ] ++ lib.optionals backupCfg.enable [
+      "d ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
+      "Z ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
+      "d ${backupCfg.stagingDir} 0700 project-zomboid project-zomboid - -"
+      "d ${backupCfg.archiveDir} 0700 project-zomboid project-zomboid - -"
     ];
 
     systemd.services.project-zomboid = {
@@ -186,6 +443,10 @@ in {
             ${pkgs.coreutils}/bin/printf '%s\n' '};';
           } > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
         ''}
+        ${lib.optionalString (cfg.sandboxProperties == { }) ''
+          ${pkgs.coreutils}/bin/rm -f \
+            ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
+        ''}
       '';
 
       serviceConfig = {
@@ -205,6 +466,34 @@ in {
       };
     };
 
+    systemd.services.project-zomboid-backup = lib.mkIf backupCfg.enable {
+      description = "No-stop Project Zomboid backup";
+      after = [ "project-zomboid.service" ];
+      unitConfig.ConditionPathExists = [
+        saveDir
+        serverConfigDir
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = "project-zomboid";
+        Group = "project-zomboid";
+        ExecStart = backupScript;
+        TimeoutStartSec = "30min";
+        UMask = "0077";
+      } // lib.optionalAttrs backupCfg.s3.enable {
+        EnvironmentFile = s3CredentialsFile;
+      };
+    };
+
+    systemd.timers.project-zomboid-backup = lib.mkIf backupCfg.enable {
+      description = "Run Project Zomboid backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = backupCfg.onCalendar;
+        Persistent = true;
+      };
+    };
+
     networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [
       cfg.port
       cfg.udpPort

nixos/system/hectic-lab/hectic-lab.nix

@@ -106,8 +106,14 @@ in {
       memory = "3g";
       serverName = "servertest";
       serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
+      backup = {
+        enable = true;
+        onCalendar = "*:0/30";
+        retentionDays = 14;
+        s3.enable = false;
+      };
       serverProperties = {
-        Map = "vehicle_interior_arcadia75;Muldraugh, KY";
+        Map = "Muldraugh, KY";
         DoLuaChecksum = false;
         Public = true;
         AntiCheatSafety = 4;
@@ -127,55 +133,43 @@ in {
         AntiCheatNoClip = 4;
         AntiCheatServerCustomization = 4;
       };
-      sandboxProperties = {
-        StartMonth = 12;
-        StartDay = 1;
-        WaterShut = 3;
-        WaterShutModifier = 60;
-        ElecShut = 3;
-        ElecShutModifier = 60;
-        MinutesPerPage = 0.5;
-        ZombieLore = {
-          Transmission = 4;
-          Mortality = 7;
-        };
-      };
       workshopItems = [
-        "3773972040" # Arcadia RV Interiors B42.20
-        "2210760610" # Cryogenic Winter +Easy/Hard Modes
         "3676456221" # Lua Digital Watch Framework
         "3600401184" # Realistic Temperature Mod
-        "3387824513" # Material Weight Reducer
-        "3413150945" # More Damaged Objects
-        "3512708849" # Shotgun Trajectory
-        "3401576145" # Firearm Models: Redux
-        "3401134276" # Vanilla Gear Expanded
-        "2956146279" # Rain Cleans Blood
-        "3693258802" # Tactical Hold
-        "3394588830" # Simple Flashlight on Belt
-        "2684285534" # Spongie's Clothing
-        "2812326159" # Spongie's Open Jackets
       ];
       mods = [
-        "\\ArcadiaRVInterior_B42_MP"
-        "\\ArcadiaRVInterior_B42_Vanilla"
-        "\\CryogenicWinter2NormalMode"
         "\\LuaDigitalWatchUI"
         "\\RC_RealisticColdMod"
-        "\\Material Weight Reducer"
-        "\\Ammunition Weight Reducer"
-        "\\MoreDamagedObjects"
-        "\\ShotgunTrajectory"
-        "\\FMR"
-        "\\VanillaGearExpanded"
-        "\\RainCleansBlood"
-        "\\TacHold Complete"
-        "\\LightOnBelt"
-        "\\SpnCloth"
-        "\\SpnOpenClothBase"
-        "\\SpnOpenCloth"
       ];
+    sandboxProperties = {
+      StartMonth = 12;
+      StartDay = 1;
+      WaterShut = 3;
+      WaterShutModifier = 150;
+      ElecShut = 3;
+      ElecShutModifier = 150;
+      MinutesPerPage = 0.5;
+      Zombies = 4;
+      ZombieConfig = {
+        PopulationMultiplier = 1.3;
+        PopulationStartMultiplier = 1.0;
+        PopulationPeakMultiplier = 1.0;
+        RespawnHours = 0.0;
+        RespawnUnseenHours = 0.0;
+        RespawnMultiplier = 0.0;
+        RedistributeHours = 0.0;
+      };
+      ZombieLore = {
+        Transmission = 4;
+        Mortality = 7;
+        Speed = 2;
+        SprinterPercentage = 0;
+        Strength = 2;
+        Cognition = 2;
+        DoorOpeningPercentage = 10;
+      };
     };
+  };
     services.p4d = {
       enable = true;
       package = pkgs.p4d;
@@ -253,8 +247,8 @@ in {
         key = "init-postgresql";
       };
       "atticd/environment" = {};
-      "immich/storage-box" = {};
-      "wg-bfs/private-key" = {};
+       "immich/storage-box" = {};
+       "wg-bfs/private-key" = {};
       "gitea-runner/org-registration-token" = {
         sopsFile = flake + "/sus/gitea-runners.yaml";
         key      = "gitea/hectic-lab/org-runner-registration-token";

nixos/system/neuro/default.nix

@@ -17,10 +17,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
     ];
     config.allowUnfreePredicate = pkg:
       self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
-      "minecraft-server"
-      "neoforge"
+       "minecraft-server"
+       "neoforge"
+       "steamcmd"
+       "steam-unwrapped"
 
-      "nvidia-x11"
+       "nvidia-x11"
     ];
     # jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
     config.permittedInsecurePackages = [

nixos/system/neuro/minecraft/wow-mine-map.nix

@@ -43,7 +43,7 @@
       } >> server.properties
     '';
     enable = true;
-    jvmOpts = "-Xmx8G -Xms2G";
+    jvmOpts = "-Xmx24G -Xms2G";
     # WorldOfSosal client and server use the same pinned NeoForge.
     package = pkgs.minecraftServers.neoforge-1_21_1.override (
       builtins.fromJSON (builtins.readFile ./neoforge-21.1.250.json)

nixos/system/neuro/neuro.nix

@@ -139,7 +139,7 @@ in {
   };
 
   services.nginx = {
-    enable = true;
+    enable = false;
     virtualHosts."bfs.band" = let
       site = pkgs.runCommand "bfs-band-site" {} ''
         mkdir -p $out
@@ -224,6 +224,70 @@ in {
     archetype.dev.enable  = true;
   };
 
+  hectic.services."project-zomboid" = {
+    enable = true;
+    memory = "8g";
+    serverName = "servertest";
+    serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
+    serverProperties = {
+      Map = "Muldraugh, KY";
+      DoLuaChecksum = false;
+      Public = true;
+      AntiCheatSafety = 4;
+      AntiCheatMovement = 4;
+      AntiCheatSpeed = 4;
+      AntiCheatHit = 4;
+      AntiCheatPacket = 4;
+      AntiCheatPacketException = 4;
+      AntiCheatPermission = 4;
+      AntiCheatXP = 4;
+      AntiCheatFire = 4;
+      AntiCheatSafeHouse = 4;
+      AntiCheatRecipe = 4;
+      AntiCheatPlayer = 4;
+      AntiCheatChecksum = 4;
+      AntiCheatItem = 4;
+      AntiCheatNoClip = 4;
+      AntiCheatServerCustomization = 4;
+    };
+    workshopItems = [
+      "3676456221" # Lua Digital Watch Framework
+      "3600401184" # Realistic Temperature Mod
+    ];
+    mods = [
+      "\\LuaDigitalWatchUI"
+      "\\RC_RealisticColdMod"
+    ];
+    sandboxProperties = {
+      Zombies = 6;
+      ZombieConfig = {
+        PopulationMultiplier = 0.0;
+        PopulationStartMultiplier = 0.0;
+        PopulationPeakMultiplier = 0.0;
+        RespawnHours = 0.0;
+        RespawnUnseenHours = 0.0;
+        RespawnMultiplier = 0.0;
+        RedistributeHours = 0.0;
+      };
+    };
+  };
+
+  systemd.services.project-zomboid.preStart = lib.mkBefore ''
+    password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"}
+    properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"}
+
+    if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then
+      umask 077
+      ${pkgs.openssl}/bin/openssl rand -hex 24 > "$password_file"
+    fi
+    ${pkgs.coreutils}/bin/chmod 0600 "$password_file"
+
+    properties_file_tmp="$( ${pkgs.coreutils}/bin/mktemp "$(dirname "$properties_file")/.server-password.ini.XXXXXX")"
+    ${pkgs.coreutils}/bin/printf 'Password=%s\n' "$(<"$password_file")" > "$properties_file_tmp"
+    ${pkgs.coreutils}/bin/chmod 0600 "$properties_file_tmp"
+    ${pkgs.coreutils}/bin/mv "$properties_file_tmp" "$properties_file"
+  '';
+
   sops = {
     gnupg.sshKeyPaths         = [ ];
     age.sshKeyPaths           = [ "/etc/ssh/ssh_host_ed25519_key" ];