Files
hearth/nixos/module/hectic/service/project-zomboid.nix
T
yukkop c917e4908c
runner nix smoke / nix label and flake smoke (push) Failing after 54s
feat: zomboid backups
2026-09-22 17:15:18 +00:00

503 lines
18 KiB
Nix

{ ... }:
{
pkgs,
lib,
config,
...
}: let
cfg = config.hectic.services."project-zomboid";
serverProperties = cfg.serverProperties // {
DefaultPort = cfg.port;
UDPPort = cfg.udpPort;
WorkshopItems = lib.concatStringsSep ";" cfg.workshopItems;
Mods = lib.concatStringsSep ";" cfg.mods;
};
configLines = lib.mapAttrsToList (
name: value:
"${name}=${if builtins.isBool value then lib.boolToString value else toString value}"
) serverProperties;
sandboxValueType = lib.types.oneOf [
lib.types.bool
lib.types.int
lib.types.float
lib.types.str
(lib.types.attrsOf sandboxValueType)
];
luaValue = value:
if builtins.isBool value then
lib.boolToString value
else if builtins.isInt value || builtins.isFloat value then
toString value
else if builtins.isAttrs value then
"{ ${lib.concatStringsSep " " (lib.mapAttrsToList (name: child: "[${luaValue name}] = ${luaValue child},") value)} }"
else
"\"${lib.replaceStrings [ "\\" "\"" "\n" "\r" ] [ "\\\\" "\\\"" "\\n" "\\r" ] value}\"";
sandboxConfigLines = lib.mapAttrsToList (
name: value: "[${luaValue name}] = ${luaValue value},"
) cfg.sandboxProperties;
zomboidDir = "${cfg.dataDir}/Zomboid";
adminPasswordFile = "${cfg.dataDir}/admin-password";
backupCfg = cfg.backup;
s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile;
s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint;
s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region;
saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}";
serverConfigDir = "${zomboidDir}/Server";
backupScript = pkgs.writeShellScript "project-zomboid-backup" ''
set -eu
staging_dir=${lib.escapeShellArg backupCfg.stagingDir}
archive_dir=${lib.escapeShellArg backupCfg.archiveDir}
lock_file="$archive_dir/.backup.lock"
${pkgs.coreutils}/bin/install -d -m 0700 \
"$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}" \
"$staging_dir/Zomboid/Server" \
"$archive_dir"
exec 9>"$lock_file"
if ! ${pkgs.util-linux}/bin/flock -n 9; then
${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid backup already running; skipping.' >&2
exit 0
fi
sync_staging() {
${pkgs.rsync}/bin/rsync -a --delete \
${lib.escapeShellArg "${saveDir}/"} \
"$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}/"
${pkgs.rsync}/bin/rsync -a --delete --delete-excluded \
--include=${lib.escapeShellArg "/${cfg.serverName}_SandboxVars.lua"} \
--include=${lib.escapeShellArg "/${cfg.serverName}_spawnpoints.lua"} \
--include=${lib.escapeShellArg "/${cfg.serverName}_spawnregions.lua"} \
--exclude='*' \
${lib.escapeShellArg "${serverConfigDir}/"} \
"$staging_dir/Zomboid/Server/"
}
# Second pass narrows, but cannot eliminate, live-save inconsistency.
sync_staging
${pkgs.coreutils}/bin/sleep 5
sync_staging
timestamp="$(${pkgs.coreutils}/bin/date -u +%Y%m%dT%H%M%SZ)"
archive_name="project-zomboid-${cfg.serverName}-$timestamp.tar.zst"
archive_tmp="$archive_dir/.$archive_name.tmp"
archive="$archive_dir/$archive_name"
trap '${pkgs.coreutils}/bin/rm -f "$archive_tmp"' EXIT
${pkgs.gnutar}/bin/tar \
--use-compress-program=${lib.escapeShellArg "${pkgs.zstd}/bin/zstd -T0"} \
-C "$staging_dir" -cf "$archive_tmp" Zomboid
${pkgs.coreutils}/bin/chmod 0600 "$archive_tmp"
${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive"
trap - EXIT
${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
-name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \
-mmin +${toString (backupCfg.retentionDays * 1440)} -delete
${lib.optionalString backupCfg.s3.enable ''
if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
${pkgs.coreutils}/bin/printf '%s\n' \
'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
exit 1
fi
s3_bucket=${lib.escapeShellArg s3Bucket}
s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
s3_key="''${s3_prefix:+$s3_prefix/}$archive_name"
${pkgs.awscli2}/bin/aws s3 cp "$archive" \
"s3://$s3_bucket/$s3_key" \
--endpoint-url ${lib.escapeShellArg s3Endpoint} \
--region ${lib.escapeShellArg s3Region} \
--cli-connect-timeout 30 \
--cli-read-timeout 300 \
--only-show-errors
remote_prefix="$s3_prefix"
if [ -n "$remote_prefix" ]; then
remote_prefix="$remote_prefix/"
fi
archive_prefix=${lib.escapeShellArg "project-zomboid-${cfg.serverName}-"}
remote_list="$staging_dir/.remote-objects.json"
remote_delete_dir="$staging_dir/.remote-delete"
${pkgs.awscli2}/bin/aws s3api list-objects-v2 \
--bucket "$s3_bucket" \
--prefix "$remote_prefix" \
--endpoint-url ${lib.escapeShellArg s3Endpoint} \
--region ${lib.escapeShellArg s3Region} \
--output json > "$remote_list"
${pkgs.python3}/bin/python3 - "$remote_list" "$remote_delete_dir" \
"$(( $(${pkgs.coreutils}/bin/date +%s) - ${toString (backupCfg.s3.remoteRetentionDays * 86400)} ))" \
"$remote_prefix$archive_prefix" <<'PY'
import datetime
import json
import os
import re
import sys
objects_path, delete_dir, cutoff, key_prefix = sys.argv[1:]
cutoff = int(cutoff)
archive_pattern = re.compile(
re.escape(key_prefix) + r"\d{8}T\d{6}Z\.tar\.zst\Z"
)
with open(objects_path, encoding="utf-8") as stream:
objects = json.load(stream).get("Contents", [])
old_keys = []
for item in objects:
key = item.get("Key", "")
if not archive_pattern.fullmatch(key):
continue
modified = datetime.datetime.fromisoformat(
item["LastModified"].replace("Z", "+00:00")
)
if int(modified.timestamp()) < cutoff:
old_keys.append(key)
os.makedirs(delete_dir, exist_ok=True)
for batch_number in range(0, len(old_keys), 1000):
batch = old_keys[batch_number:batch_number + 1000]
manifest_path = os.path.join(
delete_dir, f"batch-{batch_number // 1000:04d}.json"
)
with open(manifest_path, "w", encoding="utf-8") as stream:
json.dump(
{"Objects": [{"Key": key} for key in batch], "Quiet": True},
stream,
)
PY
for remote_manifest in "$remote_delete_dir"/*.json; do
[ -f "$remote_manifest" ] || continue
${pkgs.awscli2}/bin/aws s3api delete-objects \
--bucket "$s3_bucket" \
--delete "file://$remote_manifest" \
--endpoint-url ${lib.escapeShellArg s3Endpoint} \
--region ${lib.escapeShellArg s3Region} \
--only-show-errors
done
${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir"
''}
'';
startScript = pkgs.writeShellScript "project-zomboid-start" ''
admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile})
exec ${pkgs.steam-run}/bin/steam-run \
${lib.escapeShellArg "${cfg.installDir}/start-server.sh"} \
-servername ${lib.escapeShellArg cfg.serverName} \
-adminpassword "$admin_password"
'';
in {
options.hectic.services."project-zomboid" = {
enable = lib.mkEnableOption "Project Zomboid dedicated server";
serverName = lib.mkOption {
type = lib.types.strMatching "[A-Za-z0-9._-]+";
default = "servertest";
description = "Name used for Project Zomboid server and save files.";
};
dataDir = lib.mkOption {
type = lib.types.path;
default = "/var/lib/project-zomboid";
description = "Writable state directory for the server.";
};
installDir = lib.mkOption {
type = lib.types.path;
default = "/var/lib/project-zomboid/server";
description = "Directory where SteamCMD installs the dedicated server.";
};
port = lib.mkOption {
type = lib.types.port;
default = 16261;
description = "Project Zomboid UDP discovery port.";
};
udpPort = lib.mkOption {
type = lib.types.port;
default = 16262;
description = "Project Zomboid UDP game port.";
};
branch = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Steam beta branch, for example legacy41.";
};
memory = lib.mkOption {
type = lib.types.strMatching "[0-9]+[mMgG]";
default = "3g";
description = "Maximum Java heap for the server, for example 3g.";
};
workshopItems = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Steam Workshop item IDs, downloaded and enabled by the server.";
};
mods = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Project Zomboid mod loading IDs from mod.info.";
};
serverProperties = lib.mkOption {
type = lib.types.attrsOf (
lib.types.oneOf [
lib.types.bool
lib.types.int
lib.types.str
]
);
default = { };
description = "Additional or overriding values for the server INI file.";
};
serverPropertiesFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "Runtime file with additional INI values, suitable for secrets.";
};
sandboxProperties = lib.mkOption {
type = lib.types.attrsOf sandboxValueType;
default = { };
description = "Values for the Project Zomboid SandboxVars.lua file.";
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Open the Project Zomboid UDP ports in the firewall.";
};
backup = {
enable = lib.mkEnableOption "no-stop Project Zomboid backups";
onCalendar = lib.mkOption {
type = lib.types.str;
default = "*:0/30";
description = "systemd calendar expression controlling backup frequency.";
};
stagingDir = lib.mkOption {
type = lib.types.path;
default = "${cfg.dataDir}/backups/staging";
description = "Local directory containing the two-pass rsync staging tree.";
};
archiveDir = lib.mkOption {
type = lib.types.path;
default = "${cfg.dataDir}/backups/archive";
description = "Local directory containing timestamped tar.zst archives.";
};
retentionDays = lib.mkOption {
type = lib.types.ints.positive;
default = 14;
description = "Delete local archives older than this many days.";
};
s3 = {
enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage";
credentialsFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Runtime env file containing AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY. Required when S3 upload is enabled.
'';
};
bucket = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "S3 bucket receiving backup archives.";
};
endpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "S3-compatible endpoint URL.";
};
region = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "S3 region passed to awscli2.";
};
prefix = lib.mkOption {
type = lib.types.str;
default = "project-zomboid";
description = "Optional object key prefix within the S3 bucket.";
};
remoteRetentionDays = lib.mkOption {
type = lib.types.ints.positive;
default = 14;
description = "Delete uploaded archives older than this many days.";
};
};
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = !backupCfg.s3.enable || backupCfg.enable;
message = "hectic.services.project-zomboid.backup must be enabled before S3 upload.";
}
{
assertion = !backupCfg.s3.enable || backupCfg.s3.credentialsFile != null;
message = "hectic.services.project-zomboid.backup.s3.credentialsFile is required when S3 upload is enabled.";
}
{
assertion = !backupCfg.s3.enable || backupCfg.s3.bucket != null;
message = "hectic.services.project-zomboid.backup.s3.bucket is required when S3 upload is enabled.";
}
{
assertion = !backupCfg.s3.enable || backupCfg.s3.endpoint != null;
message = "hectic.services.project-zomboid.backup.s3.endpoint is required when S3 upload is enabled.";
}
{
assertion = !backupCfg.s3.enable || backupCfg.s3.region != null;
message = "hectic.services.project-zomboid.backup.s3.region is required when S3 upload is enabled.";
}
{
assertion =
!backupCfg.s3.enable
|| backupCfg.s3.credentialsFile == null
|| (
lib.hasPrefix "/" backupCfg.s3.credentialsFile
&& !lib.hasPrefix "/nix/store/" backupCfg.s3.credentialsFile
);
message = "hectic.services.project-zomboid.backup.s3.credentialsFile must be a runtime path outside /nix/store.";
}
{
assertion =
!backupCfg.s3.enable
|| backupCfg.s3.endpoint == null
|| lib.hasPrefix "https://" backupCfg.s3.endpoint;
message = "hectic.services.project-zomboid.backup.s3.endpoint must use HTTPS.";
}
];
users.groups.project-zomboid = { };
users.users.project-zomboid = {
isSystemUser = true;
group = "project-zomboid";
home = cfg.dataDir;
createHome = true;
};
systemd.tmpfiles.rules = [
"d ${cfg.dataDir} 0750 project-zomboid project-zomboid - -"
"d ${cfg.installDir} 0750 project-zomboid project-zomboid - -"
] ++ lib.optionals backupCfg.enable [
"d ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
"Z ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
"d ${backupCfg.stagingDir} 0700 project-zomboid project-zomboid - -"
"d ${backupCfg.archiveDir} 0700 project-zomboid project-zomboid - -"
];
systemd.services.project-zomboid = {
description = "Project Zomboid dedicated server";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
preStart = ''
${pkgs.coreutils}/bin/install -d -m 0750 \
${lib.escapeShellArg cfg.installDir}
if [ ! -s ${lib.escapeShellArg adminPasswordFile} ]; then
umask 077
${pkgs.openssl}/bin/openssl rand -base64 32 > ${lib.escapeShellArg adminPasswordFile}
fi
${pkgs.steamcmd}/bin/steamcmd \
+force_install_dir ${lib.escapeShellArg cfg.installDir} \
+login anonymous \
+app_update 380870 ${lib.optionalString (cfg.branch != null) "-beta ${lib.escapeShellArg cfg.branch}"} validate \
+quit
${pkgs.gnused}/bin/sed -i -E \
's/"-Xmx[0-9]+[mMgG]"/"-Xmx${cfg.memory}"/' \
${lib.escapeShellArg "${cfg.installDir}/ProjectZomboid64.json"}
${pkgs.coreutils}/bin/install -d -m 0750 \
${lib.escapeShellArg "${zomboidDir}/Server"}
{
${lib.concatMapStringsSep "\n " (line:
"${pkgs.coreutils}/bin/printf '%s\\n' ${lib.escapeShellArg line};"
) configLines}
${lib.optionalString (cfg.serverPropertiesFile != null)
"${pkgs.coreutils}/bin/cat ${lib.escapeShellArg cfg.serverPropertiesFile};"}
} > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}.ini"}
${lib.optionalString (cfg.sandboxProperties != { }) ''
{
${pkgs.coreutils}/bin/printf '%s\n' 'SandboxVars = {';
${lib.concatMapStringsSep "\n " (line:
"${pkgs.coreutils}/bin/printf '%s\\n' ${lib.escapeShellArg line};"
) sandboxConfigLines}
${pkgs.coreutils}/bin/printf '%s\n' '};';
} > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
''}
${lib.optionalString (cfg.sandboxProperties == { }) ''
${pkgs.coreutils}/bin/rm -f \
${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
''}
'';
serviceConfig = {
User = "project-zomboid";
Group = "project-zomboid";
WorkingDirectory = cfg.dataDir;
Environment = [
"HOME=${cfg.dataDir}"
"SteamAppId=108600"
];
ExecStart = startScript;
Restart = "on-failure";
RestartSec = 5;
TimeoutStartSec = "15min";
TimeoutStopSec = 30;
UMask = "0077";
};
};
systemd.services.project-zomboid-backup = lib.mkIf backupCfg.enable {
description = "No-stop Project Zomboid backup";
after = [ "project-zomboid.service" ];
unitConfig.ConditionPathExists = [
saveDir
serverConfigDir
];
serviceConfig = {
Type = "oneshot";
User = "project-zomboid";
Group = "project-zomboid";
ExecStart = backupScript;
TimeoutStartSec = "30min";
UMask = "0077";
} // lib.optionalAttrs backupCfg.s3.enable {
EnvironmentFile = s3CredentialsFile;
};
};
systemd.timers.project-zomboid-backup = lib.mkIf backupCfg.enable {
description = "Run Project Zomboid backup";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = backupCfg.onCalendar;
Persistent = true;
};
};
networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [
cfg.port
cfg.udpPort
];
};
}