feat: update pz

docs/project-zomboid-backups.md

@@ -10,7 +10,13 @@ or pausing the server. The default schedule is every 30 minutes. Each run:
    `Zomboid/Server` into a private staging tree;
 3. waits five seconds and repeats the rsync to narrow the live-write window;
 4. publishes a timestamped `tar.zst` archive; and
-5. deletes local archives older than `backup.retentionDays`.
+5. uploads it to S3 when enabled, then applies local retention.
+
+With `backup.retentionDays = 0`, local timestamped archives are removed after a
+successful S3 upload (or after local creation when S3 is disabled). A failed
+upload leaves the current archive locally and the next run retries that archive
+before creating a new one. Restore archives named `pre-restore` are not managed
+by this cleanup.
 
 The service lock prevents overlapping runs. Missing save or server-config paths
 skip the run through systemd `ConditionPathExists` checks.
@@ -29,12 +35,18 @@ after a restore.
 
 ## hectic-lab
 
-hectic-lab runs the timer every 30 minutes and keeps local archives for 14 days:
+hectic-lab runs the timer every 30 minutes and stores timestamped backups in
+S3 for 14 days. It keeps no regular timestamped backup archives locally:
 
 ```text
 /var/lib/project-zomboid/backups/archive/
 ```
 
+The existing `project-zomboid-restore.sh` helper expects the fresh backup it
+creates to remain in this directory for rollback. Therefore, with local
+retention set to zero, do not use the helper until it is adapted for S3-only
+retention; temporarily configure positive local retention for a restore.
+
 Check it with:
 
 ```sh
@@ -71,6 +83,17 @@ available; it remains the stronger recovery and cleanup control.
 Restoring must be done while the server is stopped so it cannot modify files
 during extraction:
 
+On hectic-lab, regular timestamped archives are retained only in S3. With
+valid S3 credentials, download the chosen archive before restoring it:
+
+```sh
+aws s3 cp \
+  s3://backup-hectic-lab/project-zomboid/project-zomboid-servertest-<timestamp>.tar.zst \
+  /var/lib/project-zomboid/backups/archive/<archive>.tar.zst \
+  --endpoint-url https://hel1.your-objectstorage.com \
+  --region hel1
+```
+
 The versioned helper creates a fresh current-state backup, stops the timer and
 server, validates archive paths, restores the save, and starts both services:
 

nixos/module/hectic/service/project-zomboid.nix

@@ -43,6 +43,7 @@
   s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
   s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint;
   s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region;
+  timestampArchivePattern = "project-zomboid-${cfg.serverName}-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z.tar.zst";
   saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}";
   serverConfigDir = "${zomboidDir}/Server";
   backupScript = pkgs.writeShellScript "project-zomboid-backup" ''
@@ -63,6 +64,49 @@
       exit 0
     fi
 
+    ${lib.optionalString backupCfg.s3.enable ''
+      if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
+        ${pkgs.coreutils}/bin/printf '%s\n' \
+          'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
+        exit 1
+      fi
+      s3_bucket=${lib.escapeShellArg s3Bucket}
+      s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
+      s3_upload() {
+        source=$1
+        key=$2
+        attempt=1
+        while [ "$attempt" -le 3 ]; do
+          if ${pkgs.awscli2}/bin/aws s3 cp "$source" "s3://$s3_bucket/$key" \
+            --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+            --region ${lib.escapeShellArg s3Region} \
+            --cli-connect-timeout 30 \
+            --cli-read-timeout 300 \
+            --only-show-errors; then
+            return 0
+          fi
+          if [ "$attempt" -eq 3 ]; then
+            return 1
+          fi
+          ${pkgs.coreutils}/bin/sleep 5
+          attempt=$((attempt + 1))
+        done
+      }
+    ''}
+
+    ${lib.optionalString (backupCfg.s3.enable && backupCfg.retentionDays == 0) ''
+      pending_list="$staging_dir/.pending-archives"
+      ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+        -name ${lib.escapeShellArg timestampArchivePattern} \
+        -printf '%T@ %p\n' | ${pkgs.coreutils}/bin/sort -n > "$pending_list"
+      while IFS= read -r pending_line; do
+        pending_archive="''${pending_line#* }"
+        pending_name="''${pending_archive##*/}"
+        s3_upload "$pending_archive" "''${s3_prefix:+$s3_prefix/}$pending_name"
+        ${pkgs.coreutils}/bin/rm -f "$pending_archive"
+      done < "$pending_list"
+      ${pkgs.coreutils}/bin/rm -f "$pending_list"
+    ''}
     ${lib.optionalString cfg.rcon.enable ''
       rcon_password="$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})"
       if [ -z "$rcon_password" ]; then
@@ -107,26 +151,17 @@
     ${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive"
     trap - EXIT
 
-    ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
-      -name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \
-      -mmin +${toString (backupCfg.retentionDays * 1440)} -delete
-
     ${lib.optionalString backupCfg.s3.enable ''
-      if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
-        ${pkgs.coreutils}/bin/printf '%s\n' \
-          'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
-        exit 1
-      fi
-      s3_bucket=${lib.escapeShellArg s3Bucket}
-      s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
       s3_key="''${s3_prefix:+$s3_prefix/}$archive_name"
-      ${pkgs.awscli2}/bin/aws s3 cp "$archive" \
-        "s3://$s3_bucket/$s3_key" \
-        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
-        --region ${lib.escapeShellArg s3Region} \
-        --cli-connect-timeout 30 \
-        --cli-read-timeout 300 \
-        --only-show-errors
+      s3_upload "$archive" "$s3_key"
+      ${lib.optionalString (backupCfg.retentionDays == 0) ''
+        ${pkgs.coreutils}/bin/rm -f "$archive"
+      ''}
+      ${lib.optionalString (backupCfg.retentionDays != 0) ''
+        ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+          -name ${lib.escapeShellArg timestampArchivePattern} \
+          -mmin +${toString (backupCfg.retentionDays * 1440)} -delete
+      ''}
 
       remote_prefix="$s3_prefix"
       if [ -n "$remote_prefix" ]; then
@@ -192,6 +227,18 @@ PY
       done
       ${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir"
     ''}
+
+    ${lib.optionalString (!backupCfg.s3.enable) ''
+      if [ ${toString (if backupCfg.retentionDays == 0 then 0 else 1)} -eq 0 ]; then
+        ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+          -name ${lib.escapeShellArg timestampArchivePattern} \
+          -delete
+      else
+        ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+          -name ${lib.escapeShellArg timestampArchivePattern} \
+          -mmin +${toString (backupCfg.retentionDays * 1440)} -delete
+      fi
+    ''}
   '';
   startScript = pkgs.writeShellScript "project-zomboid-start" ''
     admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile})
@@ -326,9 +373,9 @@ in {
       };
 
       retentionDays = lib.mkOption {
-        type = lib.types.ints.positive;
+        type = lib.types.ints.between 0 3650;
         default = 14;
-        description = "Delete local archives older than this many days.";
+        description = "Delete local archives older than this many days; zero keeps no local archives.";
       };
 
       saveWaitSeconds = lib.mkOption {

nixos/system/hectic-lab/hectic-lab.nix

@@ -110,7 +110,7 @@ in {
       backup = {
         enable = true;
         onCalendar = "*:0/30";
-        retentionDays = 14;
+        retentionDays = 0;
         s3 = {
           enable = true;
           bucket = "backup-hectic-lab";