From 3f7856b4aa5ebfac6883468fda04ae30a44d9f82 Mon Sep 17 00:00:00 2001 From: yukkop Date: Mon, 28 Sep 2026 14:50:05 +0000 Subject: [PATCH] feat: update pz --- docs/project-zomboid-backups.md | 27 +++++- .../module/hectic/service/project-zomboid.nix | 87 ++++++++++++++----- nixos/system/hectic-lab/hectic-lab.nix | 2 +- 3 files changed, 93 insertions(+), 23 deletions(-) diff --git a/docs/project-zomboid-backups.md b/docs/project-zomboid-backups.md index 9ec187af..8f705cb3 100644 --- a/docs/project-zomboid-backups.md +++ b/docs/project-zomboid-backups.md @@ -10,7 +10,13 @@ or pausing the server. The default schedule is every 30 minutes. Each run: `Zomboid/Server` into a private staging tree; 3. waits five seconds and repeats the rsync to narrow the live-write window; 4. publishes a timestamped `tar.zst` archive; and -5. deletes local archives older than `backup.retentionDays`. +5. uploads it to S3 when enabled, then applies local retention. + +With `backup.retentionDays = 0`, local timestamped archives are removed after a +successful S3 upload (or after local creation when S3 is disabled). A failed +upload leaves the current archive locally and the next run retries that archive +before creating a new one. Restore archives named `pre-restore` are not managed +by this cleanup. The service lock prevents overlapping runs. Missing save or server-config paths skip the run through systemd `ConditionPathExists` checks. @@ -29,12 +35,18 @@ after a restore. ## hectic-lab -hectic-lab runs the timer every 30 minutes and keeps local archives for 14 days: +hectic-lab runs the timer every 30 minutes and stores timestamped backups in +S3 for 14 days. It keeps no regular timestamped backup archives locally: ```text /var/lib/project-zomboid/backups/archive/ ``` +The existing `project-zomboid-restore.sh` helper expects the fresh backup it +creates to remain in this directory for rollback. Therefore, with local +retention set to zero, do not use the helper until it is adapted for S3-only +retention; temporarily configure positive local retention for a restore. + Check it with: ```sh @@ -71,6 +83,17 @@ available; it remains the stronger recovery and cleanup control. Restoring must be done while the server is stopped so it cannot modify files during extraction: +On hectic-lab, regular timestamped archives are retained only in S3. With +valid S3 credentials, download the chosen archive before restoring it: + +```sh +aws s3 cp \ + s3://backup-hectic-lab/project-zomboid/project-zomboid-servertest-.tar.zst \ + /var/lib/project-zomboid/backups/archive/.tar.zst \ + --endpoint-url https://hel1.your-objectstorage.com \ + --region hel1 +``` + The versioned helper creates a fresh current-state backup, stops the timer and server, validates archive paths, restores the save, and starts both services: diff --git a/nixos/module/hectic/service/project-zomboid.nix b/nixos/module/hectic/service/project-zomboid.nix index 84e9c7e9..7b1f21d4 100644 --- a/nixos/module/hectic/service/project-zomboid.nix +++ b/nixos/module/hectic/service/project-zomboid.nix @@ -43,6 +43,7 @@ s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket; s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint; s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region; + timestampArchivePattern = "project-zomboid-${cfg.serverName}-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z.tar.zst"; saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}"; serverConfigDir = "${zomboidDir}/Server"; backupScript = pkgs.writeShellScript "project-zomboid-backup" '' @@ -63,6 +64,49 @@ exit 0 fi + ${lib.optionalString backupCfg.s3.enable '' + if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then + ${pkgs.coreutils}/bin/printf '%s\n' \ + 'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2 + exit 1 + fi + s3_bucket=${lib.escapeShellArg s3Bucket} + s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix} + s3_upload() { + source=$1 + key=$2 + attempt=1 + while [ "$attempt" -le 3 ]; do + if ${pkgs.awscli2}/bin/aws s3 cp "$source" "s3://$s3_bucket/$key" \ + --endpoint-url ${lib.escapeShellArg s3Endpoint} \ + --region ${lib.escapeShellArg s3Region} \ + --cli-connect-timeout 30 \ + --cli-read-timeout 300 \ + --only-show-errors; then + return 0 + fi + if [ "$attempt" -eq 3 ]; then + return 1 + fi + ${pkgs.coreutils}/bin/sleep 5 + attempt=$((attempt + 1)) + done + } + ''} + + ${lib.optionalString (backupCfg.s3.enable && backupCfg.retentionDays == 0) '' + pending_list="$staging_dir/.pending-archives" + ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \ + -name ${lib.escapeShellArg timestampArchivePattern} \ + -printf '%T@ %p\n' | ${pkgs.coreutils}/bin/sort -n > "$pending_list" + while IFS= read -r pending_line; do + pending_archive="''${pending_line#* }" + pending_name="''${pending_archive##*/}" + s3_upload "$pending_archive" "''${s3_prefix:+$s3_prefix/}$pending_name" + ${pkgs.coreutils}/bin/rm -f "$pending_archive" + done < "$pending_list" + ${pkgs.coreutils}/bin/rm -f "$pending_list" + ''} ${lib.optionalString cfg.rcon.enable '' rcon_password="$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})" if [ -z "$rcon_password" ]; then @@ -107,26 +151,17 @@ ${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive" trap - EXIT - ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \ - -name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \ - -mmin +${toString (backupCfg.retentionDays * 1440)} -delete - ${lib.optionalString backupCfg.s3.enable '' - if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then - ${pkgs.coreutils}/bin/printf '%s\n' \ - 'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2 - exit 1 - fi - s3_bucket=${lib.escapeShellArg s3Bucket} - s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix} s3_key="''${s3_prefix:+$s3_prefix/}$archive_name" - ${pkgs.awscli2}/bin/aws s3 cp "$archive" \ - "s3://$s3_bucket/$s3_key" \ - --endpoint-url ${lib.escapeShellArg s3Endpoint} \ - --region ${lib.escapeShellArg s3Region} \ - --cli-connect-timeout 30 \ - --cli-read-timeout 300 \ - --only-show-errors + s3_upload "$archive" "$s3_key" + ${lib.optionalString (backupCfg.retentionDays == 0) '' + ${pkgs.coreutils}/bin/rm -f "$archive" + ''} + ${lib.optionalString (backupCfg.retentionDays != 0) '' + ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \ + -name ${lib.escapeShellArg timestampArchivePattern} \ + -mmin +${toString (backupCfg.retentionDays * 1440)} -delete + ''} remote_prefix="$s3_prefix" if [ -n "$remote_prefix" ]; then @@ -192,6 +227,18 @@ PY done ${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir" ''} + + ${lib.optionalString (!backupCfg.s3.enable) '' + if [ ${toString (if backupCfg.retentionDays == 0 then 0 else 1)} -eq 0 ]; then + ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \ + -name ${lib.escapeShellArg timestampArchivePattern} \ + -delete + else + ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \ + -name ${lib.escapeShellArg timestampArchivePattern} \ + -mmin +${toString (backupCfg.retentionDays * 1440)} -delete + fi + ''} ''; startScript = pkgs.writeShellScript "project-zomboid-start" '' admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile}) @@ -326,9 +373,9 @@ in { }; retentionDays = lib.mkOption { - type = lib.types.ints.positive; + type = lib.types.ints.between 0 3650; default = 14; - description = "Delete local archives older than this many days."; + description = "Delete local archives older than this many days; zero keeps no local archives."; }; saveWaitSeconds = lib.mkOption { diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index 7b000819..aedce9cd 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -110,7 +110,7 @@ in { backup = { enable = true; onCalendar = "*:0/30"; - retentionDays = 14; + retentionDays = 0; s3 = { enable = true; bucket = "backup-hectic-lab";