feat: zomboid backups

nixos/module/hectic/service/project-zomboid.nix

@@ -37,6 +37,147 @@
   ) cfg.sandboxProperties;
   zomboidDir = "${cfg.dataDir}/Zomboid";
   adminPasswordFile = "${cfg.dataDir}/admin-password";
+  backupCfg = cfg.backup;
+  s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile;
+  s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
+  s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint;
+  s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region;
+  saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}";
+  serverConfigDir = "${zomboidDir}/Server";
+  backupScript = pkgs.writeShellScript "project-zomboid-backup" ''
+    set -eu
+
+    staging_dir=${lib.escapeShellArg backupCfg.stagingDir}
+    archive_dir=${lib.escapeShellArg backupCfg.archiveDir}
+    lock_file="$archive_dir/.backup.lock"
+
+    ${pkgs.coreutils}/bin/install -d -m 0700 \
+      "$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}" \
+      "$staging_dir/Zomboid/Server" \
+      "$archive_dir"
+
+    exec 9>"$lock_file"
+    if ! ${pkgs.util-linux}/bin/flock -n 9; then
+      ${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid backup already running; skipping.' >&2
+      exit 0
+    fi
+
+    sync_staging() {
+      ${pkgs.rsync}/bin/rsync -a --delete \
+        ${lib.escapeShellArg "${saveDir}/"} \
+        "$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}/"
+      ${pkgs.rsync}/bin/rsync -a --delete --delete-excluded \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_SandboxVars.lua"} \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_spawnpoints.lua"} \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_spawnregions.lua"} \
+        --exclude='*' \
+        ${lib.escapeShellArg "${serverConfigDir}/"} \
+        "$staging_dir/Zomboid/Server/"
+    }
+
+    # Second pass narrows, but cannot eliminate, live-save inconsistency.
+    sync_staging
+    ${pkgs.coreutils}/bin/sleep 5
+    sync_staging
+
+    timestamp="$(${pkgs.coreutils}/bin/date -u +%Y%m%dT%H%M%SZ)"
+    archive_name="project-zomboid-${cfg.serverName}-$timestamp.tar.zst"
+    archive_tmp="$archive_dir/.$archive_name.tmp"
+    archive="$archive_dir/$archive_name"
+    trap '${pkgs.coreutils}/bin/rm -f "$archive_tmp"' EXIT
+    ${pkgs.gnutar}/bin/tar \
+      --use-compress-program=${lib.escapeShellArg "${pkgs.zstd}/bin/zstd -T0"} \
+      -C "$staging_dir" -cf "$archive_tmp" Zomboid
+    ${pkgs.coreutils}/bin/chmod 0600 "$archive_tmp"
+    ${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive"
+    trap - EXIT
+
+    ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+      -name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \
+      -mmin +${toString (backupCfg.retentionDays * 1440)} -delete
+
+    ${lib.optionalString backupCfg.s3.enable ''
+      if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
+        ${pkgs.coreutils}/bin/printf '%s\n' \
+          'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
+        exit 1
+      fi
+      s3_bucket=${lib.escapeShellArg s3Bucket}
+      s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
+      s3_key="''${s3_prefix:+$s3_prefix/}$archive_name"
+      ${pkgs.awscli2}/bin/aws s3 cp "$archive" \
+        "s3://$s3_bucket/$s3_key" \
+        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+        --region ${lib.escapeShellArg s3Region} \
+        --cli-connect-timeout 30 \
+        --cli-read-timeout 300 \
+        --only-show-errors
+
+      remote_prefix="$s3_prefix"
+      if [ -n "$remote_prefix" ]; then
+        remote_prefix="$remote_prefix/"
+      fi
+      archive_prefix=${lib.escapeShellArg "project-zomboid-${cfg.serverName}-"}
+      remote_list="$staging_dir/.remote-objects.json"
+      remote_delete_dir="$staging_dir/.remote-delete"
+      ${pkgs.awscli2}/bin/aws s3api list-objects-v2 \
+        --bucket "$s3_bucket" \
+        --prefix "$remote_prefix" \
+        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+        --region ${lib.escapeShellArg s3Region} \
+        --output json > "$remote_list"
+      ${pkgs.python3}/bin/python3 - "$remote_list" "$remote_delete_dir" \
+        "$(( $(${pkgs.coreutils}/bin/date +%s) - ${toString (backupCfg.s3.remoteRetentionDays * 86400)} ))" \
+        "$remote_prefix$archive_prefix" <<'PY'
+import datetime
+import json
+import os
+import re
+import sys
+
+objects_path, delete_dir, cutoff, key_prefix = sys.argv[1:]
+cutoff = int(cutoff)
+archive_pattern = re.compile(
+    re.escape(key_prefix) + r"\d{8}T\d{6}Z\.tar\.zst\Z"
+)
+with open(objects_path, encoding="utf-8") as stream:
+    objects = json.load(stream).get("Contents", [])
+
+old_keys = []
+for item in objects:
+    key = item.get("Key", "")
+    if not archive_pattern.fullmatch(key):
+        continue
+    modified = datetime.datetime.fromisoformat(
+        item["LastModified"].replace("Z", "+00:00")
+    )
+    if int(modified.timestamp()) < cutoff:
+        old_keys.append(key)
+
+os.makedirs(delete_dir, exist_ok=True)
+for batch_number in range(0, len(old_keys), 1000):
+    batch = old_keys[batch_number:batch_number + 1000]
+    manifest_path = os.path.join(
+        delete_dir, f"batch-{batch_number // 1000:04d}.json"
+    )
+    with open(manifest_path, "w", encoding="utf-8") as stream:
+        json.dump(
+            {"Objects": [{"Key": key} for key in batch], "Quiet": True},
+            stream,
+        )
+PY
+      for remote_manifest in "$remote_delete_dir"/*.json; do
+        [ -f "$remote_manifest" ] || continue
+        ${pkgs.awscli2}/bin/aws s3api delete-objects \
+          --bucket "$s3_bucket" \
+          --delete "file://$remote_manifest" \
+          --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+          --region ${lib.escapeShellArg s3Region} \
+          --only-show-errors
+      done
+      ${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir"
+    ''}
+  '';
   startScript = pkgs.writeShellScript "project-zomboid-start" ''
     admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile})
     exec ${pkgs.steam-run}/bin/steam-run \
@@ -131,9 +272,120 @@ in {
       default = true;
       description = "Open the Project Zomboid UDP ports in the firewall.";
     };
+
+    backup = {
+      enable = lib.mkEnableOption "no-stop Project Zomboid backups";
+
+      onCalendar = lib.mkOption {
+        type = lib.types.str;
+        default = "*:0/30";
+        description = "systemd calendar expression controlling backup frequency.";
+      };
+
+      stagingDir = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/backups/staging";
+        description = "Local directory containing the two-pass rsync staging tree.";
+      };
+
+      archiveDir = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/backups/archive";
+        description = "Local directory containing timestamped tar.zst archives.";
+      };
+
+      retentionDays = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 14;
+        description = "Delete local archives older than this many days.";
+      };
+
+      s3 = {
+        enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage";
+
+        credentialsFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            Runtime env file containing AWS_ACCESS_KEY_ID and
+            AWS_SECRET_ACCESS_KEY. Required when S3 upload is enabled.
+          '';
+        };
+
+        bucket = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3 bucket receiving backup archives.";
+        };
+
+        endpoint = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3-compatible endpoint URL.";
+        };
+
+        region = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3 region passed to awscli2.";
+        };
+
+        prefix = lib.mkOption {
+          type = lib.types.str;
+          default = "project-zomboid";
+          description = "Optional object key prefix within the S3 bucket.";
+        };
+
+        remoteRetentionDays = lib.mkOption {
+          type = lib.types.ints.positive;
+          default = 14;
+          description = "Delete uploaded archives older than this many days.";
+        };
+      };
+    };
   };
 
   config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.enable;
+        message = "hectic.services.project-zomboid.backup must be enabled before S3 upload.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.credentialsFile != null;
+        message = "hectic.services.project-zomboid.backup.s3.credentialsFile is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.bucket != null;
+        message = "hectic.services.project-zomboid.backup.s3.bucket is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.endpoint != null;
+        message = "hectic.services.project-zomboid.backup.s3.endpoint is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.region != null;
+        message = "hectic.services.project-zomboid.backup.s3.region is required when S3 upload is enabled.";
+      }
+      {
+        assertion =
+          !backupCfg.s3.enable
+          || backupCfg.s3.credentialsFile == null
+          || (
+            lib.hasPrefix "/" backupCfg.s3.credentialsFile
+            && !lib.hasPrefix "/nix/store/" backupCfg.s3.credentialsFile
+          );
+        message = "hectic.services.project-zomboid.backup.s3.credentialsFile must be a runtime path outside /nix/store.";
+      }
+      {
+        assertion =
+          !backupCfg.s3.enable
+          || backupCfg.s3.endpoint == null
+          || lib.hasPrefix "https://" backupCfg.s3.endpoint;
+        message = "hectic.services.project-zomboid.backup.s3.endpoint must use HTTPS.";
+      }
+    ];
+
     users.groups.project-zomboid = { };
     users.users.project-zomboid = {
       isSystemUser = true;
@@ -145,6 +397,11 @@ in {
     systemd.tmpfiles.rules = [
       "d ${cfg.dataDir} 0750 project-zomboid project-zomboid - -"
       "d ${cfg.installDir} 0750 project-zomboid project-zomboid - -"
+    ] ++ lib.optionals backupCfg.enable [
+      "d ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
+      "Z ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
+      "d ${backupCfg.stagingDir} 0700 project-zomboid project-zomboid - -"
+      "d ${backupCfg.archiveDir} 0700 project-zomboid project-zomboid - -"
     ];
 
     systemd.services.project-zomboid = {
@@ -186,6 +443,10 @@ in {
             ${pkgs.coreutils}/bin/printf '%s\n' '};';
           } > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
         ''}
+        ${lib.optionalString (cfg.sandboxProperties == { }) ''
+          ${pkgs.coreutils}/bin/rm -f \
+            ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
+        ''}
       '';
 
       serviceConfig = {
@@ -205,6 +466,34 @@ in {
       };
     };
 
+    systemd.services.project-zomboid-backup = lib.mkIf backupCfg.enable {
+      description = "No-stop Project Zomboid backup";
+      after = [ "project-zomboid.service" ];
+      unitConfig.ConditionPathExists = [
+        saveDir
+        serverConfigDir
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = "project-zomboid";
+        Group = "project-zomboid";
+        ExecStart = backupScript;
+        TimeoutStartSec = "30min";
+        UMask = "0077";
+      } // lib.optionalAttrs backupCfg.s3.enable {
+        EnvironmentFile = s3CredentialsFile;
+      };
+    };
+
+    systemd.timers.project-zomboid-backup = lib.mkIf backupCfg.enable {
+      description = "Run Project Zomboid backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = backupCfg.onCalendar;
+        Persistent = true;
+      };
+    };
+
     networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [
       cfg.port
       cfg.udpPort