Compare commits
38 Commits
3299daf061
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 557b6e9ef0 | |||
| e49f497045 | |||
| feb1a48db1 | |||
| 30732080b7 | |||
| 80cf1588bb | |||
| ef7d1b29f4 | |||
| 7e8c6884db | |||
| 1dd41e608b | |||
| e41c3e5a05 | |||
| f473280bf5 | |||
| b08fdd6e6b | |||
| f73bfc63be | |||
| f6e7c1eca9 | |||
| 6996d178ef | |||
| 7f7229b199 | |||
| 2d5bd26c36 | |||
| fba150b55b | |||
| 2e7bf58acf | |||
| b12c35f957 | |||
| bcf1b84dc4 | |||
| 7c25e3b46d | |||
| a20381e343 | |||
| bd92610a98 | |||
| e3ee881db6 | |||
| b9eabca464 | |||
| fcc72192f5 | |||
| 5a0696ce64 | |||
| f4a59ff117 | |||
| 129c82c863 | |||
| 968c654320 | |||
| 98bb6c568f | |||
| 72168aa8fa | |||
| 28dde5b9b1 | |||
| c2e0ba200c | |||
| 2eb23ea7ea | |||
| 9906f71c5d | |||
| 593c0d9abc | |||
| 53dfd60b4c |
@@ -0,0 +1,29 @@
|
|||||||
|
name: runner nix smoke
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- .gitea/workflows/runner-nix-smoke.yaml
|
||||||
|
- flake.lock
|
||||||
|
- flake.nix
|
||||||
|
- infra/gitea-runners/**
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: nix label smoke
|
||||||
|
runs-on: nix
|
||||||
|
steps:
|
||||||
|
- name: Nix version and cache configuration
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
nix --version
|
||||||
|
nix config show substituters
|
||||||
|
nix config show trusted-public-keys
|
||||||
|
|
||||||
|
- name: Repository flake evaluation
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
nix flake check --no-build
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
name: runner ubuntu smoke
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- .gitea/workflows/runner-ubuntu-smoke.yaml
|
||||||
|
- infra/gitea-runners/**
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: ubuntu-latest label smoke
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Basic runner information
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
echo "hello from gitea runner"
|
||||||
|
uname -a
|
||||||
|
|
||||||
|
- name: Docker smoke when available
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
|
||||||
|
docker run --rm hello-world
|
||||||
|
else
|
||||||
|
echo "docker command not available; skipping Docker smoke"
|
||||||
|
fi
|
||||||
@@ -47,6 +47,15 @@ creation_rules:
|
|||||||
- *hectic-lab-server
|
- *hectic-lab-server
|
||||||
- *umbriel-bfs
|
- *umbriel-bfs
|
||||||
|
|
||||||
|
- path_regex: sus/gitea-runners.yaml$
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *nrv
|
||||||
|
- *yukkop
|
||||||
|
- *yukkop-alt
|
||||||
|
- *hectic-lab-server
|
||||||
|
- *umbriel-bfs
|
||||||
|
|
||||||
- path_regex: sus/matrix-cluster.yaml$
|
- path_regex: sus/matrix-cluster.yaml$
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
flake,
|
||||||
|
self,
|
||||||
|
inputs,
|
||||||
|
system ? "aarch64-darwin",
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
name = builtins.baseNameOf ./.;
|
||||||
|
in inputs.nix-darwin.lib.darwinSystem {
|
||||||
|
inherit system;
|
||||||
|
specialArgs = { inherit flake self inputs; };
|
||||||
|
modules = [
|
||||||
|
inputs.home-manager.darwinModules.home-manager
|
||||||
|
{
|
||||||
|
networking.hostName = name;
|
||||||
|
nixpkgs.hostPlatform = system;
|
||||||
|
nixpkgs.overlays = [ self.overlays.default ];
|
||||||
|
}
|
||||||
|
./${name}.nix
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
{
|
||||||
|
flake,
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
name = "yukkop";
|
||||||
|
in {
|
||||||
|
system.primaryUser = name;
|
||||||
|
nix.settings.experimental-features = "nix-command flakes";
|
||||||
|
|
||||||
|
programs.zsh.enable = true;
|
||||||
|
|
||||||
|
services.openssh.enable = true;
|
||||||
|
|
||||||
|
users.users.${name} = {
|
||||||
|
home = "/Users/${name}";
|
||||||
|
openssh.authorizedKeys.keys = [
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = with pkgs; [
|
||||||
|
aerospace
|
||||||
|
git
|
||||||
|
moreutils
|
||||||
|
neovim
|
||||||
|
tmux
|
||||||
|
];
|
||||||
|
|
||||||
|
launchd.user.agents.aerospace = {
|
||||||
|
serviceConfig = {
|
||||||
|
ProgramArguments = [
|
||||||
|
"${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
|
||||||
|
];
|
||||||
|
RunAtLoad = true;
|
||||||
|
KeepAlive = true;
|
||||||
|
StandardOutPath = "/tmp/aerospace.out.log";
|
||||||
|
StandardErrorPath = "/tmp/aerospace.err.log";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
home-manager.useGlobalPkgs = true;
|
||||||
|
home-manager.useUserPackages = true;
|
||||||
|
home-manager.backupFileExtension = "backup";
|
||||||
|
home-manager.sharedModules = [
|
||||||
|
(flake + "/home/module/program/tmux.nix")
|
||||||
|
];
|
||||||
|
home-manager.users.${name} = {
|
||||||
|
home.stateVersion = "25.11";
|
||||||
|
|
||||||
|
home.packages = with pkgs; [
|
||||||
|
iproute2mac
|
||||||
|
jujutsu
|
||||||
|
ripgrep
|
||||||
|
];
|
||||||
|
|
||||||
|
programs.git = {
|
||||||
|
enable = true;
|
||||||
|
lfs.enable = true;
|
||||||
|
settings = {
|
||||||
|
user.name = name;
|
||||||
|
user.email = "hectic.yukkop@gmail.com";
|
||||||
|
push.autoSetupRemote = true;
|
||||||
|
init.defaultBranch = "master";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
programs.zsh = {
|
||||||
|
enable = true;
|
||||||
|
enableCompletion = true;
|
||||||
|
autosuggestion.enable = true;
|
||||||
|
syntaxHighlighting.enable = true;
|
||||||
|
|
||||||
|
history = {
|
||||||
|
size = 10000;
|
||||||
|
path = "$HOME/.zsh/.zsh_history";
|
||||||
|
};
|
||||||
|
|
||||||
|
shellAliases = {
|
||||||
|
drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
|
||||||
|
nv = "nvim";
|
||||||
|
tmux = "tmux a";
|
||||||
|
};
|
||||||
|
|
||||||
|
initContent = ''
|
||||||
|
export PATH=/Users/yukkop/.opencode/bin:$PATH
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
xdg.configFile."aerospace/aerospace.toml".text = ''
|
||||||
|
start-at-login = false
|
||||||
|
|
||||||
|
enable-normalization-flatten-containers = true
|
||||||
|
enable-normalization-opposite-orientation-for-nested-containers = true
|
||||||
|
|
||||||
|
default-root-container-layout = 'tiles'
|
||||||
|
default-root-container-orientation = 'auto'
|
||||||
|
accordion-padding = 30
|
||||||
|
|
||||||
|
on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
|
||||||
|
automatically-unhide-macos-hidden-apps = false
|
||||||
|
|
||||||
|
[exec]
|
||||||
|
inherit-env-vars = true
|
||||||
|
|
||||||
|
[exec.env-vars]
|
||||||
|
PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
|
||||||
|
|
||||||
|
[gaps]
|
||||||
|
inner.horizontal = 8
|
||||||
|
inner.vertical = 8
|
||||||
|
outer.left = 8
|
||||||
|
outer.bottom = 8
|
||||||
|
outer.top = 8
|
||||||
|
outer.right = 8
|
||||||
|
|
||||||
|
[mode.main.binding]
|
||||||
|
alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
|
||||||
|
|
||||||
|
alt-slash = 'layout tiles horizontal vertical'
|
||||||
|
alt-comma = 'layout accordion horizontal vertical'
|
||||||
|
alt-f = 'fullscreen'
|
||||||
|
|
||||||
|
alt-h = 'focus left'
|
||||||
|
alt-j = 'focus down'
|
||||||
|
alt-k = 'focus up'
|
||||||
|
alt-l = 'focus right'
|
||||||
|
|
||||||
|
alt-shift-h = 'move left'
|
||||||
|
alt-shift-j = 'move down'
|
||||||
|
alt-shift-k = 'move up'
|
||||||
|
alt-shift-l = 'move right'
|
||||||
|
|
||||||
|
alt-minus = 'resize smart -50'
|
||||||
|
alt-equal = 'resize smart +50'
|
||||||
|
|
||||||
|
alt-1 = 'workspace 1'
|
||||||
|
alt-2 = 'workspace 2'
|
||||||
|
alt-3 = 'workspace 3'
|
||||||
|
alt-4 = 'workspace 4'
|
||||||
|
alt-5 = 'workspace 5'
|
||||||
|
alt-6 = 'workspace 6'
|
||||||
|
alt-7 = 'workspace 7'
|
||||||
|
alt-8 = 'workspace 8'
|
||||||
|
alt-9 = 'workspace 9'
|
||||||
|
|
||||||
|
alt-shift-1 = 'move-node-to-workspace 1'
|
||||||
|
alt-shift-2 = 'move-node-to-workspace 2'
|
||||||
|
alt-shift-3 = 'move-node-to-workspace 3'
|
||||||
|
alt-shift-4 = 'move-node-to-workspace 4'
|
||||||
|
alt-shift-5 = 'move-node-to-workspace 5'
|
||||||
|
alt-shift-6 = 'move-node-to-workspace 6'
|
||||||
|
alt-shift-7 = 'move-node-to-workspace 7'
|
||||||
|
alt-shift-8 = 'move-node-to-workspace 8'
|
||||||
|
alt-shift-9 = 'move-node-to-workspace 9'
|
||||||
|
|
||||||
|
alt-tab = 'workspace-back-and-forth'
|
||||||
|
alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
|
||||||
|
|
||||||
|
alt-shift-semicolon = 'mode service'
|
||||||
|
|
||||||
|
[mode.service.binding]
|
||||||
|
esc = ['reload-config', 'mode main']
|
||||||
|
r = ['flatten-workspace-tree', 'mode main']
|
||||||
|
f = ['layout floating tiling', 'mode main']
|
||||||
|
b = ['balance-sizes', 'mode main']
|
||||||
|
backspace = ['close-all-windows-but-current', 'mode main']
|
||||||
|
|
||||||
|
alt-shift-h = ['join-with left', 'mode main']
|
||||||
|
alt-shift-j = ['join-with down', 'mode main']
|
||||||
|
alt-shift-k = ['join-with up', 'mode main']
|
||||||
|
alt-shift-l = ['join-with right', 'mode main']
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
system.stateVersion = 6;
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
haskell = import ./haskell.nix { inherit self system pkgs; };
|
haskell = import ./haskell.nix { inherit self system pkgs; };
|
||||||
neuro = import ./neuro.nix { inherit self system pkgs; };
|
neuro = import ./neuro.nix { inherit self system pkgs; };
|
||||||
xmpp = import ./xmpp.nix { inherit self system pkgs; };
|
xmpp = import ./xmpp.nix { inherit self system pkgs; };
|
||||||
|
gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
|
||||||
default = pkgs.mkShell {
|
default = pkgs.mkShell {
|
||||||
buildInputs =
|
buildInputs =
|
||||||
(with self.packages.${system}; [
|
(with self.packages.${system}; [
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
{ pkgs, ... }: let
|
||||||
|
opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
|
||||||
|
|
||||||
|
tofu = pkgs.writeShellScriptBin "tofu" ''
|
||||||
|
exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
|
||||||
|
'';
|
||||||
|
|
||||||
|
giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
|
||||||
|
cat <<'EOF'
|
||||||
|
Gitea runners setup checklist
|
||||||
|
|
||||||
|
Tools available in this shell:
|
||||||
|
tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
|
||||||
|
docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
|
||||||
|
|
||||||
|
Environment expected before real deploy/apply:
|
||||||
|
TF_VAR_hcloud_token
|
||||||
|
TF_VAR_ssh_public_key
|
||||||
|
TF_VAR_ssh_private_key
|
||||||
|
S3 backend credentials and endpoint access
|
||||||
|
a matching SOPS age identity for sus/gitea-runners.yaml
|
||||||
|
kubectl access to the target cluster
|
||||||
|
a concrete registry digest for the pushed Nix-capable runner image if enabling
|
||||||
|
the nix label
|
||||||
|
|
||||||
|
OpenTofu validation gate:
|
||||||
|
tofu version
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
|
||||||
|
Nix image build/publish/digest gate:
|
||||||
|
nix build .#gitea-runner-nix-image
|
||||||
|
publish the archive, then pin the registry-reported digest in the runner label
|
||||||
|
mapping
|
||||||
|
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
|
||||||
|
SOPS token Secret creation gate:
|
||||||
|
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
|
||||||
|
Cluster provision gate:
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||||
|
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||||
|
|
||||||
|
Kubernetes apply gate:
|
||||||
|
kubectl config current-context
|
||||||
|
kubectl get nodes -o wide
|
||||||
|
kubectl get sc
|
||||||
|
kubectl apply -k infra/gitea-runners/k8s
|
||||||
|
|
||||||
|
Verification commands:
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
|
||||||
|
Main blockers and gates:
|
||||||
|
do not run tofu apply without all external inputs
|
||||||
|
do not apply the k8s overlay until the gitea-runner-token Secret exists
|
||||||
|
do not enable the nix label until the image has been published with a concrete digest
|
||||||
|
do not print, load, or require secrets on shell entry
|
||||||
|
EOF
|
||||||
|
'';
|
||||||
|
in pkgs.mkShell {
|
||||||
|
name = "gitea-runners";
|
||||||
|
|
||||||
|
buildInputs = [
|
||||||
|
tofu
|
||||||
|
giteaRunnersSetup
|
||||||
|
pkgs.nix
|
||||||
|
pkgs.kubectl
|
||||||
|
pkgs.kustomize
|
||||||
|
pkgs.kubeconform
|
||||||
|
pkgs.sops
|
||||||
|
pkgs.age
|
||||||
|
pkgs.awscli2
|
||||||
|
pkgs.hcloud
|
||||||
|
pkgs.tea
|
||||||
|
pkgs.docker
|
||||||
|
pkgs.skopeo
|
||||||
|
pkgs.go-containerregistry
|
||||||
|
pkgs.jq
|
||||||
|
pkgs.yq-go
|
||||||
|
pkgs.curl
|
||||||
|
pkgs.git
|
||||||
|
pkgs.openssh
|
||||||
|
];
|
||||||
|
|
||||||
|
shellHook = ''
|
||||||
|
export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
|
||||||
|
export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
|
||||||
|
export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
|
||||||
|
export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
|
||||||
|
export GITEA_RUNNERS_NAMESPACE="gitea-runners"
|
||||||
|
|
||||||
|
alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
|
||||||
|
alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
|
||||||
|
alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Gitea runner setup DevShell ==="
|
||||||
|
echo ""
|
||||||
|
echo "Run gitea-runners-setup for the full setup checklist."
|
||||||
|
echo "Paths: "
|
||||||
|
echo " root=$GITEA_RUNNERS_ROOT"
|
||||||
|
echo " tofu=$GITEA_RUNNERS_TOFU_DIR"
|
||||||
|
echo " k8s=$GITEA_RUNNERS_K8S_DIR"
|
||||||
|
echo " image=$GITEA_RUNNERS_IMAGE_DIR"
|
||||||
|
echo ""
|
||||||
|
'';
|
||||||
|
}
|
||||||
Generated
+26
-4
@@ -675,11 +675,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1780435429,
|
"lastModified": 1780905541,
|
||||||
"narHash": "sha256-J69BMplOQOalw/RfY7NP7x4kd1Pozm5h4Ie8PpCc4gg=",
|
"narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
|
||||||
"ref": "refs/heads/master",
|
"ref": "refs/heads/master",
|
||||||
"rev": "4f03d4f0ddc8483a180190828f6c756e428f6011",
|
"rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
|
||||||
"revCount": 113,
|
"revCount": 123,
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||||
},
|
},
|
||||||
@@ -688,6 +688,27 @@
|
|||||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"nix-darwin": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1772129556,
|
||||||
|
"narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
|
||||||
|
"owner": "nix-darwin",
|
||||||
|
"repo": "nix-darwin",
|
||||||
|
"rev": "ebec37af18215214173c98cf6356d0aca24a2585",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-darwin",
|
||||||
|
"ref": "nix-darwin-25.11",
|
||||||
|
"repo": "nix-darwin",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nix-minecraft": {
|
"nix-minecraft": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"flake-compat": "flake-compat_3",
|
"flake-compat": "flake-compat_3",
|
||||||
@@ -967,6 +988,7 @@
|
|||||||
"hyprland": "hyprland",
|
"hyprland": "hyprland",
|
||||||
"impermanence": "impermanence",
|
"impermanence": "impermanence",
|
||||||
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
|
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
|
||||||
|
"nix-darwin": "nix-darwin",
|
||||||
"nix-minecraft": "nix-minecraft",
|
"nix-minecraft": "nix-minecraft",
|
||||||
"nixos-anywhere": "nixos-anywhere",
|
"nixos-anywhere": "nixos-anywhere",
|
||||||
"nixos-hardware": "nixos-hardware",
|
"nixos-hardware": "nixos-hardware",
|
||||||
|
|||||||
@@ -42,6 +42,10 @@
|
|||||||
url = "github:nix-community/home-manager/release-25.11";
|
url = "github:nix-community/home-manager/release-25.11";
|
||||||
inputs.nixpkgs.follows = "nixpkgs";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
|
nix-darwin = {
|
||||||
|
url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
nixos-wsl = {
|
nixos-wsl = {
|
||||||
url = "github:nix-community/NixOS-WSL";
|
url = "github:nix-community/NixOS-WSL";
|
||||||
inputs.nixpkgs.follows = "nixpkgs";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
@@ -119,8 +123,12 @@
|
|||||||
# FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this
|
# FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this
|
||||||
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
|
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
|
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
|
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
|
"tenix|x86_64-linux" = import ./nixos/system/tenix { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
|
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
|
};
|
||||||
|
darwinConfigurations = {
|
||||||
|
"yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{ pkgs, ... }: {
|
||||||
|
programs.tmux = {
|
||||||
|
enable = true;
|
||||||
|
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
|
||||||
|
keyMode = "vi";
|
||||||
|
escapeTime = 500;
|
||||||
|
historyLimit = 50000;
|
||||||
|
newSession = true;
|
||||||
|
extraConfig = ''
|
||||||
|
# resurrect
|
||||||
|
set -g @resurrect-strategy-vim 'session'
|
||||||
|
set -g @resurrect-strategy-nvim 'session'
|
||||||
|
set -g @resurrect-capture-pane-contents 'on'
|
||||||
|
|
||||||
|
resurrect_dir="$HOME/.tmux/resurrect"
|
||||||
|
set -g @resurrect-dir $resurrect_dir
|
||||||
|
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
|
||||||
|
|
||||||
|
# continuum
|
||||||
|
set -g @continuum-restore 'on'
|
||||||
|
set -g @continuum-boot 'on'
|
||||||
|
set -g @continuum-save-interval '10'
|
||||||
|
|
||||||
|
bind-key -T copy-mode-vi v send-keys -X begin-selection
|
||||||
|
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
|
||||||
|
|
||||||
|
bind-key O select-pane -t :.-
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
# Gitea runner Nix image
|
||||||
|
|
||||||
|
The repo-owned Nix-capable job image is built by the flake package
|
||||||
|
`gitea-runner-nix-image`.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#gitea-runner-nix-image
|
||||||
|
```
|
||||||
|
|
||||||
|
The package emits a Docker archive with the local build tag:
|
||||||
|
|
||||||
|
```text
|
||||||
|
gitea-runner-nix-image:2026-06-07
|
||||||
|
```
|
||||||
|
|
||||||
|
That tag is build metadata only. Do not use it as the final Gitea runner label
|
||||||
|
mapping because runner job images must be immutable.
|
||||||
|
|
||||||
|
## Publication target
|
||||||
|
|
||||||
|
Preferred registry:
|
||||||
|
|
||||||
|
```text
|
||||||
|
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish the archive without adding secrets to the image layers, then use the
|
||||||
|
registry-reported digest as the only final `nix` label image reference:
|
||||||
|
|
||||||
|
```text
|
||||||
|
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
```
|
||||||
|
|
||||||
|
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
|
||||||
|
runner label mapping must use the `@sha256:` reference above. Keep
|
||||||
|
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
|
||||||
|
configuration task explicitly changes it. Only the `nix` label should select
|
||||||
|
this custom image.
|
||||||
|
|
||||||
|
If the Gitea container registry is unavailable, select a private registry that
|
||||||
|
is reachable from the runner Kubernetes cluster and requires authentication that
|
||||||
|
can be provided through Kubernetes image-pull secrets. Record the selected
|
||||||
|
registry and replace the host in the same digest-pinned form:
|
||||||
|
|
||||||
|
```text
|
||||||
|
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not fall back to `latest` or a tag-only mapping.
|
||||||
|
|
||||||
|
## Task 7 publication status
|
||||||
|
|
||||||
|
Local build evidence is recorded in
|
||||||
|
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
|
||||||
|
load and tag the image, but pushing to the preferred registry failed with
|
||||||
|
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
|
||||||
|
concrete final mapping. Kubernetes pull smoke is recorded in
|
||||||
|
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
|
||||||
|
is not installed or not on `PATH`.
|
||||||
|
|
||||||
|
Once registry credentials are available, rerun the push, capture the
|
||||||
|
registry-reported digest, and replace `<registry-digest>` in the mapping above
|
||||||
|
before Task 6/9 consumes the label configuration.
|
||||||
|
|
||||||
|
## Image contents
|
||||||
|
|
||||||
|
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
|
||||||
|
`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
|
||||||
|
top-level `flake.nix`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
experimental-features = nix-command flakes
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||||
|
sandbox = false
|
||||||
|
```
|
||||||
|
|
||||||
|
No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
|
||||||
|
credential belongs in this image. Runtime secrets stay with the Kubernetes
|
||||||
|
runner configuration and token-file mount contract.
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- persistentvolumeclaims
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- apiGroups:
|
||||||
|
- apps
|
||||||
|
resources:
|
||||||
|
- statefulsets
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
data:
|
||||||
|
cleanup-dry-run.sh: |
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
namespace="${RUNNER_NAMESPACE:-gitea-runners}"
|
||||||
|
mode="${CLEANUP_MODE:-dry-run}"
|
||||||
|
selector="app.kubernetes.io/name=gitea-runner"
|
||||||
|
|
||||||
|
if [ "$namespace" != "gitea-runners" ]; then
|
||||||
|
printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
|
||||||
|
exit 13
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$mode" != "dry-run" ]; then
|
||||||
|
printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
|
||||||
|
exit 13
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'gitea runner lifecycle cleanup dry-run\n'
|
||||||
|
printf 'namespace: %s\n' "$namespace"
|
||||||
|
printf 'mode: %s\n\n' "$mode"
|
||||||
|
|
||||||
|
printf 'StatefulSet:\n'
|
||||||
|
kubectl -n "$namespace" get statefulset gitea-runner -o wide
|
||||||
|
|
||||||
|
printf '\nActive runner pods:\n'
|
||||||
|
kubectl -n "$namespace" get pods -l "$selector" -o wide
|
||||||
|
|
||||||
|
printf '\nRunner /data PVCs:\n'
|
||||||
|
kubectl -n "$namespace" get pvc -l "$selector" -o wide
|
||||||
|
|
||||||
|
printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
|
||||||
|
found_candidate=0
|
||||||
|
for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
pod="${pvc#data-}"
|
||||||
|
if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
|
||||||
|
found_candidate=1
|
||||||
|
printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$found_candidate" -eq 0 ]; then
|
||||||
|
printf 'none\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '\nGitea registration reconciliation:\n'
|
||||||
|
printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
|
||||||
|
printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-cleanup-dry-run
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
schedule: "17 3 * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
successfulJobsHistoryLimit: 3
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 3600
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
serviceAccountName: gitea-runner-lifecycle
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: cleanup-dry-run
|
||||||
|
image: bitnami/kubectl:1.30
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- /scripts/cleanup-dry-run.sh
|
||||||
|
env:
|
||||||
|
- name: RUNNER_NAMESPACE
|
||||||
|
value: gitea-runners
|
||||||
|
- name: CLEANUP_MODE
|
||||||
|
value: dry-run
|
||||||
|
volumeMounts:
|
||||||
|
- name: lifecycle-scripts
|
||||||
|
mountPath: /scripts
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: lifecycle-scripts
|
||||||
|
configMap:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
defaultMode: 0555
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
- service.yaml
|
||||||
|
- service-account.yaml
|
||||||
|
- runner-config.yaml
|
||||||
|
- statefulset.yaml
|
||||||
|
- cleanup-lifecycle.yaml
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-config
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
data:
|
||||||
|
config.yaml: |
|
||||||
|
log:
|
||||||
|
level: info
|
||||||
|
|
||||||
|
runner:
|
||||||
|
file: /data/.runner
|
||||||
|
capacity: 1
|
||||||
|
envs: {}
|
||||||
|
timeout: 3h
|
||||||
|
insecure: false
|
||||||
|
fetch_timeout: 5s
|
||||||
|
fetch_interval: 2s
|
||||||
|
labels:
|
||||||
|
- ubuntu-latest
|
||||||
|
# The nix label is intentionally disabled until the runner image has a
|
||||||
|
# concrete registry-reported digest; see ../runbook.md before deploy.
|
||||||
|
|
||||||
|
cache:
|
||||||
|
enabled: true
|
||||||
|
dir: /data/cache
|
||||||
|
|
||||||
|
container:
|
||||||
|
network: bridge
|
||||||
|
privileged: false
|
||||||
|
force_pull: true
|
||||||
|
valid_volumes: []
|
||||||
|
docker_host: unix:///runner-docker/docker.sock
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
rules: []
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: gitea-runner
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
ports:
|
||||||
|
- name: cache
|
||||||
|
port: 8088
|
||||||
|
targetPort: cache
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
serviceName: gitea-runner
|
||||||
|
replicas: 5
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
annotations:
|
||||||
|
hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
|
||||||
|
spec:
|
||||||
|
serviceAccountName: gitea-runner
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
securityContext:
|
||||||
|
fsGroup: 1000
|
||||||
|
containers:
|
||||||
|
- name: runner
|
||||||
|
image: gitea/act_runner:0.2.11
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: GITEA_INSTANCE_URL
|
||||||
|
value: https://gitea.hectic-lab.com
|
||||||
|
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
||||||
|
value: /runner-secrets/token
|
||||||
|
- name: CONFIG_FILE
|
||||||
|
value: /runner-config/config.yaml
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: unix:///runner-docker/docker.sock
|
||||||
|
ports:
|
||||||
|
- name: cache
|
||||||
|
containerPort: 8088
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- test -s /data/.runner && test -S /runner-docker/docker.sock
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 6
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- test -S /runner-docker/docker.sock
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 6
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
- name: config
|
||||||
|
mountPath: /runner-config
|
||||||
|
readOnly: true
|
||||||
|
- name: runner-token
|
||||||
|
mountPath: /runner-secrets
|
||||||
|
readOnly: true
|
||||||
|
- name: docker-socket
|
||||||
|
mountPath: /runner-docker
|
||||||
|
- name: docker
|
||||||
|
image: docker:27-dind
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- --host=unix:///runner-docker/docker.sock
|
||||||
|
- --storage-driver=overlay2
|
||||||
|
- --tls=false
|
||||||
|
env:
|
||||||
|
- name: DOCKER_TLS_CERTDIR
|
||||||
|
value: ""
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: unix:///runner-docker/docker.sock
|
||||||
|
securityContext:
|
||||||
|
# Privileged rootful DinD is intentionally scoped to this trusted
|
||||||
|
# internal runner pool; never expose it to untrusted fork/PR jobs.
|
||||||
|
privileged: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 4Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- docker
|
||||||
|
- info
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 6
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- docker
|
||||||
|
- info
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 6
|
||||||
|
volumeMounts:
|
||||||
|
- name: docker-socket
|
||||||
|
mountPath: /runner-docker
|
||||||
|
- name: docker-graph
|
||||||
|
mountPath: /var/lib/docker
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: gitea-runner-config
|
||||||
|
- name: runner-token
|
||||||
|
secret:
|
||||||
|
secretName: gitea-runner-token
|
||||||
|
items:
|
||||||
|
- key: token
|
||||||
|
path: token
|
||||||
|
defaultMode: 0400
|
||||||
|
- name: docker-socket
|
||||||
|
emptyDir: {}
|
||||||
|
- name: docker-graph
|
||||||
|
emptyDir: {}
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: hcloud-volumes
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# OpenTofu working directory and downloaded modules/providers.
|
||||||
|
.terraform/
|
||||||
|
.terraform.lock.hcl
|
||||||
|
|
||||||
|
# State must live in the S3 backend for production. Local state is allowed only
|
||||||
|
# for throwaway syntax checks with `tofu init -backend=false` and must not be
|
||||||
|
# committed.
|
||||||
|
terraform.tfstate
|
||||||
|
terraform.tfstate.*
|
||||||
|
*.tfstate
|
||||||
|
*.tfstate.*
|
||||||
|
crash.log
|
||||||
|
crash.*.log
|
||||||
|
|
||||||
|
# Plans can contain secrets or derived infrastructure data.
|
||||||
|
*.tfplan
|
||||||
|
*.plan
|
||||||
|
kubeconfig
|
||||||
|
kubeconfig.yaml
|
||||||
|
*_kubeconfig.yaml
|
||||||
|
|
||||||
|
# Variable files commonly carry credentials. Keep production inputs in SOPS or
|
||||||
|
# external environment/configuration, not in checked-in files.
|
||||||
|
*.tfvars
|
||||||
|
*.tfvars.json
|
||||||
|
override.tf
|
||||||
|
override.tf.json
|
||||||
|
*_override.tf
|
||||||
|
*_override.tf.json
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# Gitea runner OpenTofu backend contract
|
||||||
|
|
||||||
|
This directory defines the safe backend, provider contract, and kube-hetzner
|
||||||
|
cluster stack for the Gitea runner Kubernetes cluster.
|
||||||
|
|
||||||
|
## Required backend
|
||||||
|
|
||||||
|
Production state must use the OpenTofu S3 backend in `backend.tf`:
|
||||||
|
|
||||||
|
- bucket: `gitea-runner-hectic-lab`
|
||||||
|
- key: `gitea-runners/kube-hetzner/terraform.tfstate`
|
||||||
|
- region: `fsn1`, aligned with the target Hetzner location
|
||||||
|
- encryption: `encrypt = true`
|
||||||
|
- locking: `use_lockfile = true` where the selected S3-compatible endpoint
|
||||||
|
supports it
|
||||||
|
|
||||||
|
Before any production `tofu init`, verify the S3-compatible endpoint, credential
|
||||||
|
source, bucket versioning, encryption behavior, and lockfile support for the
|
||||||
|
chosen object-storage provider. Keep backend authentication externalized through
|
||||||
|
environment variables, AWS-compatible shared config, or the production secret
|
||||||
|
injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
|
||||||
|
tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
|
||||||
|
OpenTofu files.
|
||||||
|
|
||||||
|
## Local state safety
|
||||||
|
|
||||||
|
Production local state is forbidden. Only syntax-only validation/prototyping may
|
||||||
|
use local state, and it must use backend-disabled initialization:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init -backend=false
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
```
|
||||||
|
|
||||||
|
Fail the run if production local state appears:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test ! -e infra/gitea-runners/opentofu/terraform.tfstate
|
||||||
|
test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
|
||||||
|
grep -R 'backend "s3"' infra/gitea-runners/opentofu
|
||||||
|
```
|
||||||
|
|
||||||
|
The `.gitignore` in this directory blocks local state, plans, downloaded
|
||||||
|
providers/modules, and variable files from being committed. Treat any local
|
||||||
|
state file as disposable validation residue, never as production state.
|
||||||
|
|
||||||
|
## Provider and module pins
|
||||||
|
|
||||||
|
`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
|
||||||
|
`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
|
||||||
|
observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
|
||||||
|
module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
|
||||||
|
as locals so Task 5 can wire the module without re-opening the version contract.
|
||||||
|
|
||||||
|
`providers.tf` leaves the `hcloud` provider empty so authentication comes from
|
||||||
|
the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
|
||||||
|
Do not set token values in `.tf` or `.tfvars` files.
|
||||||
|
|
||||||
|
## Cluster shape
|
||||||
|
|
||||||
|
The default cluster is deliberately fixed-size:
|
||||||
|
|
||||||
|
- cluster name: `gitea-runners`
|
||||||
|
- Hetzner location: `fsn1`
|
||||||
|
- private network region: `eu-central`
|
||||||
|
- control plane: one `cpx21` node in pool `control-plane`
|
||||||
|
- workers: three `cpx31` nodes in pool `runner-workers`
|
||||||
|
- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
|
||||||
|
- Longhorn: disabled
|
||||||
|
- autoscaling/KEDA: not enabled in this stack
|
||||||
|
|
||||||
|
The three default workers are sized for the initial five trusted privileged DinD
|
||||||
|
jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
|
||||||
|
`worker_count` to `5` or increase `worker_server_type`, then run a fresh
|
||||||
|
`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
|
||||||
|
|
||||||
|
Required inputs must come from environment or secret injection, for example
|
||||||
|
`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
|
||||||
|
Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
|
||||||
|
kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
|
||||||
|
enabled; this path is ignored as operational secret material.
|
||||||
|
|
||||||
|
## Known state caveat
|
||||||
|
|
||||||
|
kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
|
||||||
|
internal `kube_system_secrets` handling. This task does not claim that risk is
|
||||||
|
solved. Task 5 must verify the generated plan and state before production apply
|
||||||
|
and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
|
||||||
|
keys, and decrypted secrets are absent from committed files and unsafe state
|
||||||
|
evidence.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
terraform {
|
||||||
|
backend "s3" {
|
||||||
|
bucket = "gitea-runner-hectic-lab"
|
||||||
|
key = "gitea-runners/kube-hetzner/terraform.tfstate"
|
||||||
|
region = "fsn1"
|
||||||
|
encrypt = true
|
||||||
|
use_lockfile = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
check "remote_state_contract" {
|
||||||
|
assert {
|
||||||
|
condition = local.production_remote_state
|
||||||
|
error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
locals {
|
||||||
|
default_storage_class = "hcloud-volumes"
|
||||||
|
|
||||||
|
control_plane_nodepools = [
|
||||||
|
{
|
||||||
|
name = "control-plane"
|
||||||
|
server_type = var.control_plane_server_type
|
||||||
|
location = var.hetzner_location
|
||||||
|
labels = []
|
||||||
|
taints = []
|
||||||
|
count = 1
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
agent_nodepools = [
|
||||||
|
{
|
||||||
|
name = "runner-workers"
|
||||||
|
server_type = var.worker_server_type
|
||||||
|
location = var.hetzner_location
|
||||||
|
labels = ["node-role.hectic-lab/gitea-runner=true"]
|
||||||
|
taints = []
|
||||||
|
count = var.worker_count
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
module "kube_hetzner" {
|
||||||
|
source = "kube-hetzner/kube-hetzner/hcloud"
|
||||||
|
version = "2.19.3"
|
||||||
|
|
||||||
|
providers = {
|
||||||
|
hcloud = hcloud
|
||||||
|
}
|
||||||
|
|
||||||
|
hcloud_token = var.hcloud_token
|
||||||
|
ssh_public_key = var.ssh_public_key
|
||||||
|
ssh_private_key = var.ssh_private_key
|
||||||
|
|
||||||
|
cluster_name = var.cluster_name
|
||||||
|
base_domain = var.base_domain
|
||||||
|
|
||||||
|
# kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
|
||||||
|
# expose that expected path without outputting kubeconfig private key material.
|
||||||
|
create_kubeconfig = true
|
||||||
|
|
||||||
|
network_region = var.network_region
|
||||||
|
load_balancer_location = var.hetzner_location
|
||||||
|
control_plane_nodepools = local.control_plane_nodepools
|
||||||
|
agent_nodepools = local.agent_nodepools
|
||||||
|
|
||||||
|
# Hetzner CSI is the required StorageClass provider for runner PVCs.
|
||||||
|
disable_hetzner_csi = false
|
||||||
|
|
||||||
|
# Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
|
||||||
|
enable_longhorn = false
|
||||||
|
|
||||||
|
# Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
|
||||||
|
# raise worker_count to 5 or increase worker_server_type after validating pod
|
||||||
|
# CPU, memory, and ephemeral-storage pressure in Task 11.
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
output "kubeconfig_path" {
|
||||||
|
description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
|
||||||
|
value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
|
||||||
|
}
|
||||||
|
|
||||||
|
output "cluster_name" {
|
||||||
|
description = "kube-hetzner cluster name."
|
||||||
|
value = var.cluster_name
|
||||||
|
}
|
||||||
|
|
||||||
|
output "node_pool_names" {
|
||||||
|
description = "Control-plane and worker node pool names used by this stack."
|
||||||
|
value = {
|
||||||
|
control_plane = [for pool in local.control_plane_nodepools : pool.name]
|
||||||
|
workers = [for pool in local.agent_nodepools : pool.name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output "default_storage_class" {
|
||||||
|
description = "Default Hetzner CSI StorageClass expected for runner PVCs."
|
||||||
|
value = local.default_storage_class
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
provider "hcloud" {}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
variable "hcloud_token" {
|
||||||
|
description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_public_key" {
|
||||||
|
description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_private_key" {
|
||||||
|
description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "cluster_name" {
|
||||||
|
description = "Name for the kube-hetzner runner cluster."
|
||||||
|
type = string
|
||||||
|
default = "gitea-runners"
|
||||||
|
|
||||||
|
validation {
|
||||||
|
condition = can(regex("^[a-z0-9-]+$", var.cluster_name))
|
||||||
|
error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "hetzner_location" {
|
||||||
|
description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
|
||||||
|
type = string
|
||||||
|
default = "fsn1"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "network_region" {
|
||||||
|
description = "Hetzner private network region. eu-central covers fsn1."
|
||||||
|
type = string
|
||||||
|
default = "eu-central"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "control_plane_server_type" {
|
||||||
|
description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
|
||||||
|
type = string
|
||||||
|
default = "cpx21"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "worker_server_type" {
|
||||||
|
description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
|
||||||
|
type = string
|
||||||
|
default = "cpx31"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "worker_count" {
|
||||||
|
description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
|
||||||
|
type = number
|
||||||
|
default = 3
|
||||||
|
|
||||||
|
validation {
|
||||||
|
condition = var.worker_count >= 1
|
||||||
|
error_message = "worker_count must be at least 1."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "kubeconfig_path" {
|
||||||
|
description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
|
||||||
|
type = string
|
||||||
|
default = null
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "base_domain" {
|
||||||
|
description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
|
||||||
|
type = string
|
||||||
|
default = ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
terraform {
|
||||||
|
required_version = ">= 1.10.1"
|
||||||
|
|
||||||
|
required_providers {
|
||||||
|
hcloud = {
|
||||||
|
source = "hetznercloud/hcloud"
|
||||||
|
version = "1.60.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud"
|
||||||
|
kube_hetzner_module_version = "2.19.3"
|
||||||
|
hcloud_provider_minimum = ">= 1.59.0"
|
||||||
|
production_remote_state = true
|
||||||
|
}
|
||||||
@@ -0,0 +1,503 @@
|
|||||||
|
# Gitea Runner Infrastructure Runbook
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This directory is the repo-owned boundary for the first Gitea Actions runner
|
||||||
|
pool. Task 1 only establishes the scaffold and immutable decision contract;
|
||||||
|
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
|
||||||
|
and a Nix-capable runner image under the existing subdirectories.
|
||||||
|
|
||||||
|
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
|
||||||
|
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
|
||||||
|
workflow job containers are ephemeral, while each runner pod keeps its runner
|
||||||
|
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||||
|
|
||||||
|
## Immutable decisions
|
||||||
|
|
||||||
|
- Infrastructure is managed with OpenTofu command examples only, using the
|
||||||
|
`tofu` CLI.
|
||||||
|
- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
|
||||||
|
- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
|
||||||
|
- Runner implementation is the non-Enterprise `gitea/runner`.
|
||||||
|
- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
|
||||||
|
- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
|
||||||
|
file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
|
||||||
|
environment variables are not the contract.
|
||||||
|
- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
|
||||||
|
`/data`, including `/data/.runner`.
|
||||||
|
- Container builds run through privileged rootful DinD inside trusted runner
|
||||||
|
pods; host Docker socket mounting is not an implementation path.
|
||||||
|
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
|
||||||
|
the Nix-capable image has been pushed and a concrete registry-reported digest
|
||||||
|
is added to the runner ConfigMap.
|
||||||
|
- First scope is trusted internal workflows only, with no untrusted fork or PR
|
||||||
|
workflow support.
|
||||||
|
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
|
||||||
|
|
||||||
|
## Lifecycle boundaries
|
||||||
|
|
||||||
|
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
|
||||||
|
contract, Hetzner provider configuration, and kube-hetzner module wiring.
|
||||||
|
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
|
||||||
|
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
|
||||||
|
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
|
||||||
|
handoff; package or flake output changes are outside Task 1.
|
||||||
|
- `infra/gitea-runners/runbook.md`: this contract plus later operational
|
||||||
|
commands, rollback notes, and acceptance evidence references.
|
||||||
|
|
||||||
|
## Guardrails
|
||||||
|
|
||||||
|
- Enterprise ARC/actions-runner-controller are rejected alternatives and must
|
||||||
|
not be implemented here. Do not add ARC custom resources, controller install
|
||||||
|
instructions, or GitHub Actions ARC assumptions.
|
||||||
|
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
|
||||||
|
acceptable for trusted internal jobs.
|
||||||
|
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
|
||||||
|
runner pool.
|
||||||
|
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
|
||||||
|
S3 credentials, decrypted SOPS files, or SOPS age keys.
|
||||||
|
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
|
||||||
|
runner token; Kubernetes receives the token as a mounted file secret instead.
|
||||||
|
- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
|
||||||
|
jobs must reach the public HTTPS service.
|
||||||
|
|
||||||
|
## Initial acceptance commands
|
||||||
|
|
||||||
|
Run from the repository root:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
|
||||||
|
test -f infra/gitea-runners/runbook.md
|
||||||
|
grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
|
||||||
|
grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
|
||||||
|
grep -R "[t]erraform " infra/gitea-runners || true
|
||||||
|
grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected outcomes: the directory and file checks exit 0; the architecture-term
|
||||||
|
grep shows this contract; ARC references appear only in the rejected-alternative
|
||||||
|
guardrail above; there are no forbidden CLI command examples and no unresolved
|
||||||
|
decision placeholders.
|
||||||
|
|
||||||
|
## Downstream placeholders
|
||||||
|
|
||||||
|
- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
|
||||||
|
- Task 3: add SOPS secret contract and runtime token delivery details.
|
||||||
|
- Task 4: define or package the Nix-capable runner image for the `nix` label.
|
||||||
|
- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
|
||||||
|
and document cleanup, rollback, and scaling operations.
|
||||||
|
|
||||||
|
## Runner lifecycle cleanup
|
||||||
|
|
||||||
|
All lifecycle commands are scoped to the runner namespace:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
|
||||||
|
runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
|
||||||
|
does not delete pods, PVCs, Docker data, or Gitea runner registrations.
|
||||||
|
|
||||||
|
Run the same inventory on demand without waiting for the schedule:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||||
|
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||||
|
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
|
||||||
|
manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
|
||||||
|
operator to remove finished jobs manually.
|
||||||
|
|
||||||
|
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||||
|
pod loses `/data/.runner`. That runner identity must then be deregistered from
|
||||||
|
Gitea or the replacement pod must be allowed to re-register intentionally with
|
||||||
|
the current organization runner token. Do not delete an active runner PVC as a
|
||||||
|
normal cleanup step.
|
||||||
|
|
||||||
|
Non-UI Gitea registration reconciliation uses the Gitea API with a separate
|
||||||
|
admin token. Store that token outside this repository and pass it as a file; do
|
||||||
|
not print it:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||||
|
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||||
|
--restart=Never \
|
||||||
|
--image=curlimages/curl:8.10.1 \
|
||||||
|
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
|
||||||
|
has completed and its logs have been collected. Do not keep this admin token in
|
||||||
|
the runner namespace longer than the reconciliation window.
|
||||||
|
|
||||||
|
Only remove a stale Gitea runner registration after the corresponding pod/PVC
|
||||||
|
was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
|
||||||
|
Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
|
||||||
|
cleanup is a fallback, not the only path. Record the removed runner name and the
|
||||||
|
Kubernetes PVC/pod deletion that made it stale.
|
||||||
|
|
||||||
|
After the dry-run list identifies a stale registration and the PVC/pod deletion
|
||||||
|
has been recorded, remove that exact Gitea runner by id through the API:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||||
|
umask 077
|
||||||
|
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||||
|
trap 'rm -f "$curl_config"' EXIT
|
||||||
|
{
|
||||||
|
printf 'request = "DELETE"\n'
|
||||||
|
printf 'header = "Authorization: token '
|
||||||
|
cat /secure/path/gitea-admin-token
|
||||||
|
printf '"\n'
|
||||||
|
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||||
|
} > "$curl_config"
|
||||||
|
curl -fsS --config "$curl_config"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run the delete command for a runner that still has an active
|
||||||
|
`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
|
||||||
|
being intentionally reset for re-registration.
|
||||||
|
|
||||||
|
## Docker-in-Docker storage cleanup
|
||||||
|
|
||||||
|
Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
|
||||||
|
pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
|
||||||
|
list disk usage before pruning, and run the command only against the `docker`
|
||||||
|
container in runner pods in `gitea-runners`. Because this storage is pod-local,
|
||||||
|
loop over pods for pool-wide cleanup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
For one pod, replace the StatefulSet target with the pod name:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
|
||||||
|
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run host-level Docker cleanup commands and do not mount or prune a host
|
||||||
|
Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
|
||||||
|
Kubernetes; the `/data` PVC remains and still controls runner identity.
|
||||||
|
|
||||||
|
## Token rotation
|
||||||
|
|
||||||
|
Rotate the Gitea organization runner token without printing decrypted values:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sops sus/gitea-runners.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
kubectl -n gitea-runners rollout restart statefulset/gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||||
|
```
|
||||||
|
|
||||||
|
The `rollout restart` command above is the controlled restart path for this
|
||||||
|
StatefulSet. Observe the rollout and each ordinal until all replacement pods are
|
||||||
|
Ready; do not delete runner pods directly as part of normal token rotation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
Verification must confirm the token file mount remains present while the token
|
||||||
|
value never appears in logs or evidence:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deploy and status
|
||||||
|
|
||||||
|
These commands are executable only when the external inputs are available:
|
||||||
|
|
||||||
|
- `TF_VAR_hcloud_token`
|
||||||
|
- `TF_VAR_ssh_public_key`
|
||||||
|
- `TF_VAR_ssh_private_key`
|
||||||
|
- S3 backend credentials and endpoint access
|
||||||
|
- a matching SOPS age identity for `sus/gitea-runners.yaml`
|
||||||
|
- `kubectl` access to the target cluster
|
||||||
|
- a concrete digest for the pushed Nix-capable runner image, if enabling the
|
||||||
|
`nix` label
|
||||||
|
|
||||||
|
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
|
||||||
|
stale kubeconfig files.
|
||||||
|
|
||||||
|
Before production Kubernetes apply or rollout, satisfy both manifest gates:
|
||||||
|
|
||||||
|
1. Create or update the `gitea-runner-token` Secret from SOPS. The active
|
||||||
|
Kustomize overlay intentionally does not include a placeholder Secret, but
|
||||||
|
the StatefulSet still mounts `secretName: gitea-runner-token` as
|
||||||
|
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
|
||||||
|
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
|
||||||
|
image has been pushed successfully. Enable the `nix` label only by adding a
|
||||||
|
digest-pinned `docker://` mapping with the exact registry-reported sha256
|
||||||
|
digest from that push.
|
||||||
|
|
||||||
|
Use the same SOPS materialization pattern as token rotation before applying the
|
||||||
|
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
|
||||||
|
target namespace; the full overlay remains gated on the Secret and digest
|
||||||
|
decisions:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
|
||||||
|
above succeeds. Do not claim or enable the `nix` runner label until the image
|
||||||
|
publication step has produced the concrete digest.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||||
|
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||||
|
kubectl config current-context
|
||||||
|
kubectl get nodes -o wide
|
||||||
|
kubectl get sc
|
||||||
|
kubectl apply -k infra/gitea-runners/k8s
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected status after deploy:
|
||||||
|
|
||||||
|
- `kubectl config current-context` names the runner cluster context.
|
||||||
|
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
|
||||||
|
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
|
||||||
|
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
|
||||||
|
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
|
||||||
|
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
|
||||||
|
|
||||||
|
## Scale 5 to 10 to 5
|
||||||
|
|
||||||
|
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
|
||||||
|
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
|
||||||
|
# Run the concurrent smoke workflows now.
|
||||||
|
|
||||||
|
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
After scaling back down, inspect the cleanup dry-run and deregister any stale
|
||||||
|
runner registrations only for pods or PVCs that were intentionally removed.
|
||||||
|
|
||||||
|
## Cleanup and stale runner deregistration
|
||||||
|
|
||||||
|
Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
|
||||||
|
PVC candidates whose pod is gone. It must not delete active resources.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||||
|
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||||
|
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Pool-wide DinD storage checks and cleanup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||||
|
pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
|
||||||
|
replacement pod re-register intentionally with the current organization token.
|
||||||
|
Never delete an active runner PVC as routine cleanup.
|
||||||
|
|
||||||
|
Non-UI Gitea registration reconciliation uses an admin token stored outside this
|
||||||
|
repository:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||||
|
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||||
|
--restart=Never \
|
||||||
|
--image=curlimages/curl:8.10.1 \
|
||||||
|
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
|
||||||
|
pod has completed and its logs have been collected.
|
||||||
|
|
||||||
|
After the dry-run list identifies a stale registration and the PVC or pod
|
||||||
|
deletion has been recorded, remove that exact Gitea runner by id through the
|
||||||
|
API:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||||
|
umask 077
|
||||||
|
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||||
|
trap 'rm -f "$curl_config"' EXIT
|
||||||
|
{
|
||||||
|
printf 'request = "DELETE"\n'
|
||||||
|
printf 'header = "Authorization: token '
|
||||||
|
cat /secure/path/gitea-admin-token
|
||||||
|
printf '"\n'
|
||||||
|
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||||
|
} > "$curl_config"
|
||||||
|
curl -fsS --config "$curl_config"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run the delete command for a runner that still has an active
|
||||||
|
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
|
||||||
|
being intentionally reset for re-registration.
|
||||||
|
|
||||||
|
## Application rollback
|
||||||
|
|
||||||
|
Rollback the app layer only. Do not use this section to destroy the cluster.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners rollout history statefulset/gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
```
|
||||||
|
|
||||||
|
If a manifest rollback is needed, reapply the repo overlay after checking out the
|
||||||
|
known-good revision, then re-run the rollout checks:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
```
|
||||||
|
|
||||||
|
## Full cluster teardown
|
||||||
|
|
||||||
|
This destroys Hetzner resources owned by the kube-hetzner stack, including the
|
||||||
|
`gitea-runners` cluster nodes, the `control-plane` node pool, the
|
||||||
|
`runner-workers` node pool, the cluster network, load balancer resources,
|
||||||
|
firewall objects, and any attached Hetzner CSI volumes still managed by the
|
||||||
|
stack. Do not run teardown unless the destruction is intentional.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
|
||||||
|
```
|
||||||
|
|
||||||
|
## Partial OpenTofu apply recovery
|
||||||
|
|
||||||
|
If `tofu apply` fails after creating some resources, do not destroy blindly.
|
||||||
|
First reconcile state and inspect what the stack thinks exists:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu state list
|
||||||
|
```
|
||||||
|
|
||||||
|
Then rerun the normal plan path. Use `-target` only as a last resort when a
|
||||||
|
single resource is stuck and the drift is understood.
|
||||||
|
|
||||||
|
## S3 backend recovery
|
||||||
|
|
||||||
|
If backend init or state access fails, first verify the bucket and versioning
|
||||||
|
outside OpenTofu, then reconfigure the backend:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
|
||||||
|
nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan
|
||||||
|
```
|
||||||
|
|
||||||
|
If the backend reports a stale lock, confirm no `tofu` process is active, then
|
||||||
|
use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
|
||||||
|
unlock a live plan or apply.
|
||||||
|
|
||||||
|
## Gitea outage troubleshooting
|
||||||
|
|
||||||
|
Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
|
||||||
|
containers.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
```
|
||||||
|
|
||||||
|
If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
|
||||||
|
`/data/.runner` just because the service is unavailable. Once Gitea returns,
|
||||||
|
repeat the token rotation or re-registration path if a pod restarted while the
|
||||||
|
service was unavailable and lost its runner identity.
|
||||||
|
|
||||||
|
## Release checklist
|
||||||
|
|
||||||
|
Do not release unless the following evidence files exist and are readable:
|
||||||
|
|
||||||
|
- `.sisyphus/evidence/task-5-cluster-plan.txt`
|
||||||
|
- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
|
||||||
|
- `.sisyphus/evidence/task-9-deploy.txt`
|
||||||
|
- `.sisyphus/evidence/task-9-secret-mount.txt`
|
||||||
|
- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
|
||||||
|
- `.sisyphus/evidence/task-10-nix-workflow.txt`
|
||||||
|
- `.sisyphus/evidence/task-11-scale.txt`
|
||||||
|
- `.sisyphus/evidence/task-11-restart-cleanup.txt`
|
||||||
|
|
||||||
|
If any evidence file is missing, stop and collect it before treating the runbook
|
||||||
|
as complete.
|
||||||
+45
-9
@@ -26,12 +26,39 @@
|
|||||||
"aarch64-darwin"
|
"aarch64-darwin"
|
||||||
];
|
];
|
||||||
|
|
||||||
|
cudaUnfreeNames = [
|
||||||
|
"cuda_nvcc"
|
||||||
|
"cuda_cudart"
|
||||||
|
"cuda_cuobjdump"
|
||||||
|
"cuda_cupti"
|
||||||
|
"cuda_nvdisasm"
|
||||||
|
"cuda_cccl"
|
||||||
|
"cuda_nvml_dev"
|
||||||
|
"cuda_nvrtc"
|
||||||
|
"cuda_nvtx"
|
||||||
|
"cuda_profiler_api"
|
||||||
|
|
||||||
|
"libcusparse_lt"
|
||||||
|
"libcublas"
|
||||||
|
"libcufft"
|
||||||
|
"libcufile"
|
||||||
|
"libcurand"
|
||||||
|
"libcusolver"
|
||||||
|
"libnvjitlink"
|
||||||
|
"libcusparse"
|
||||||
|
"cudnn"
|
||||||
|
];
|
||||||
|
|
||||||
|
cudaUnfreePredicate = pkg:
|
||||||
|
builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
|
||||||
|
|
||||||
forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
|
forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
|
||||||
builtins.foldl' (
|
builtins.foldl' (
|
||||||
acc: system: let
|
acc: system: let
|
||||||
pkgs = import nixpkgs {
|
pkgs = import nixpkgs {
|
||||||
inherit system;
|
inherit system;
|
||||||
overlays = pkgOverlays;
|
overlays = pkgOverlays;
|
||||||
|
config.allowUnfreePredicate = cudaUnfreePredicate;
|
||||||
};
|
};
|
||||||
systemOutputs = f {
|
systemOutputs = f {
|
||||||
system = system;
|
system = system;
|
||||||
@@ -58,7 +85,7 @@
|
|||||||
else {};
|
else {};
|
||||||
in {
|
in {
|
||||||
# -- For all systems --
|
# -- For all systems --
|
||||||
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems;
|
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
|
||||||
|
|
||||||
forSystems = systems: nixpkgs.lib.genAttrs systems;
|
forSystems = systems: nixpkgs.lib.genAttrs systems;
|
||||||
forAllSystems = nixpkgs.lib.genAttrs AllSystems;
|
forAllSystems = nixpkgs.lib.genAttrs AllSystems;
|
||||||
@@ -67,6 +94,7 @@ in {
|
|||||||
logs = builtins.readFile ./shell/logs.sh;
|
logs = builtins.readFile ./shell/logs.sh;
|
||||||
check-tool = builtins.readFile ./shell/check-tool.sh;
|
check-tool = builtins.readFile ./shell/check-tool.sh;
|
||||||
local-dir = builtins.readFile ./shell/local-dir.sh;
|
local-dir = builtins.readFile ./shell/local-dir.sh;
|
||||||
|
load-sops = builtins.readFile ./shell/load-sops.sh;
|
||||||
};
|
};
|
||||||
|
|
||||||
sharedShellAliases = {
|
sharedShellAliases = {
|
||||||
@@ -102,7 +130,7 @@ in {
|
|||||||
|
|
||||||
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
|
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
|
||||||
# for everything that creates objects in the `hectic` namespace, used by
|
# for everything that creates objects in the `hectic` namespace, used by
|
||||||
# migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply
|
# migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
|
||||||
# the full bundle via lib/hook/apply-hectic-bundle.sh.
|
# the full bundle via lib/hook/apply-hectic-bundle.sh.
|
||||||
#
|
#
|
||||||
# The whole hectic system shares one `versionString`; `hectic-version.sql`
|
# The whole hectic system shares one `versionString`; `hectic-version.sql`
|
||||||
@@ -187,19 +215,27 @@ in {
|
|||||||
readModulesRecursive' = path: extraArgs:
|
readModulesRecursive' = path: extraArgs:
|
||||||
with lib;
|
with lib;
|
||||||
with builtins; let
|
with builtins; let
|
||||||
paths = pipe "${path}" [
|
collectPaths = dir: prefix:
|
||||||
(filesystem.listFilesRecursive)
|
concatLists (mapAttrsToList (name: type: let
|
||||||
(filter (hasSuffix ".nix"))
|
path' = dir + "/${name}";
|
||||||
];
|
name' = if prefix == "" then name else "${prefix}/${name}";
|
||||||
|
in
|
||||||
|
if type == "directory"
|
||||||
|
then collectPaths path' name'
|
||||||
|
else [{
|
||||||
|
inherit path';
|
||||||
|
name = name';
|
||||||
|
}]
|
||||||
|
) (readDir dir));
|
||||||
|
paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
|
||||||
pathToName = flip pipe [
|
pathToName = flip pipe [
|
||||||
(removePrefix "${path}/")
|
|
||||||
(replaceStrings ["/" ".nix"] ["." ""])
|
(replaceStrings ["/" ".nix"] ["." ""])
|
||||||
(removeSuffix ".nix")
|
(removeSuffix ".nix")
|
||||||
];
|
];
|
||||||
attrList =
|
attrList =
|
||||||
map (path': {
|
map (path': {
|
||||||
name = pathToName (unsafeDiscardStringContext path');
|
name = pathToName path'.name;
|
||||||
value = import path' extraArgs;
|
value = import path'.path' extraArgs;
|
||||||
})
|
})
|
||||||
paths;
|
paths;
|
||||||
in
|
in
|
||||||
|
|||||||
@@ -10,8 +10,8 @@
|
|||||||
# Usage:
|
# Usage:
|
||||||
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
||||||
#
|
#
|
||||||
# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via
|
# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
|
||||||
# hectic.load_secrets_from_env() after the bundle is applied.
|
# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
|
||||||
# SQL file paths are substituted by Nix evaluation time.
|
# SQL file paths are substituted by Nix evaluation time.
|
||||||
|
|
||||||
apply_hectic_bundle() {
|
apply_hectic_bundle() {
|
||||||
@@ -41,11 +41,9 @@ apply_hectic_bundle() {
|
|||||||
done
|
done
|
||||||
|
|
||||||
if [ -n "$env_content" ]; then
|
if [ -n "$env_content" ]; then
|
||||||
# Dollar-quote with $ps_env$ tag to preserve all content verbatim.
|
env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
|
||||||
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
|
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
|
||||||
SELECT hectic.load_secrets_from_env(\$ps_env\$
|
SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
|
||||||
$env_content
|
|
||||||
\$ps_env\$);
|
|
||||||
SQL
|
SQL
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ Single source of truth for every object created in the `hectic` PostgreSQL
|
|||||||
schema. Consumed by:
|
schema. Consumed by:
|
||||||
|
|
||||||
- `package/migrator` — applies the bundle on `migrator init` (mandatory).
|
- `package/migrator` — applies the bundle on `migrator init` (mandatory).
|
||||||
- `package/db-tool` — applies the bundle in `database hydrate` (default; opt
|
- `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
|
||||||
out with `--no-hook`).
|
(default; opt out with `--no-hook`) and in `db-ops secrets load`.
|
||||||
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
|
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
|
||||||
paths exposed via `self.lib.hectic.*.path`.
|
paths exposed via `self.lib.hectic.*.path`.
|
||||||
|
|
||||||
@@ -60,7 +60,7 @@ verbatim source files in the store.
|
|||||||
|
|
||||||
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
|
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
|
||||||
`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
|
`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
|
||||||
SQL paths embedded at Nix evaluation time. `migrator` and `db-tool` splice that
|
SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
|
||||||
shell source directly into their generated scripts. Public entry point:
|
shell source directly into their generated scripts. Public entry point:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -87,8 +87,9 @@ External consumers that do not want to source the helper can still invoke
|
|||||||
4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
|
4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
|
||||||
update `lib/hook/apply-hectic-bundle.sh` to apply the file.
|
update `lib/hook/apply-hectic-bundle.sh` to apply the file.
|
||||||
5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
|
5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
|
||||||
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`
|
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
|
||||||
and `test/package/db-tool/test/postgresql/hydrate-hook/`.
|
`test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
|
||||||
|
bundle-loading coverage.
|
||||||
|
|
||||||
## Versioning
|
## Versioning
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
load_sops_shell_quote() {
|
||||||
|
printf "'"
|
||||||
|
printf '%s' "$1" | sed "s/'/'\"'\"'/g"
|
||||||
|
printf "'"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_normalize_key() {
|
||||||
|
load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
|
||||||
|
|
||||||
|
case "$load_sops_normalized_key" in
|
||||||
|
''|[!A-Z_]*|*[!A-Z0-9_]*)
|
||||||
|
printf 'load-sops: invalid environment name after key normalization\n' >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
printf '%s\n' "$load_sops_normalized_key"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_env_from_sops_file() {
|
||||||
|
load_sops_file=$1
|
||||||
|
load_sops_extract=${2-}
|
||||||
|
|
||||||
|
if ! command -v sops >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `sops` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v yq >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `yq` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_decrypted=''
|
||||||
|
load_sops_attempt=0
|
||||||
|
load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
|
||||||
|
load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
|
||||||
|
|
||||||
|
while :; do
|
||||||
|
if [ -n "$load_sops_extract" ]; then
|
||||||
|
if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
|
||||||
|
load_sops_status=0
|
||||||
|
else
|
||||||
|
load_sops_status=$?
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
|
||||||
|
load_sops_status=0
|
||||||
|
else
|
||||||
|
load_sops_status=$?
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_status" -eq 0 ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_prompt" != 1 ]; then
|
||||||
|
printf 'load-sops: failed to decrypt file\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
|
||||||
|
printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_attempt=$((load_sops_attempt + 1))
|
||||||
|
if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
|
||||||
|
printf 'load-sops: decrypt failed after configured retries\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
|
||||||
|
load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
|
||||||
|
load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
|
||||||
|
case "$load_sops_use_script" in
|
||||||
|
auto)
|
||||||
|
if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
|
||||||
|
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
|
||||||
|
load_sops_script_status=0
|
||||||
|
else
|
||||||
|
load_sops_script_status=$?
|
||||||
|
fi
|
||||||
|
if [ "$load_sops_script_status" -eq 0 ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
1)
|
||||||
|
if ! command -v script >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `script` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
|
||||||
|
load_sops_script_status=0
|
||||||
|
else
|
||||||
|
load_sops_script_status=$?
|
||||||
|
fi
|
||||||
|
if [ "$load_sops_script_status" -eq 0 ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
0)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
|
||||||
|
if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
|
||||||
|
printf 'load-sops: failed to read prompt input\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
export SOPS_AGE_KEY_CMD
|
||||||
|
done
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_text "$load_sops_decrypted"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_file() {
|
||||||
|
load_sops_file=$1
|
||||||
|
|
||||||
|
if ! command -v yq >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `yq` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_text() {
|
||||||
|
load_sops_yaml=$1
|
||||||
|
load_sops_seen=''
|
||||||
|
|
||||||
|
if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
|
||||||
|
load_sops_keys_status=0
|
||||||
|
else
|
||||||
|
load_sops_keys_status=$?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_keys_status" -ne 0 ]; then
|
||||||
|
printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS= read -r load_sops_key; do
|
||||||
|
[ -n "$load_sops_key" ] || continue
|
||||||
|
|
||||||
|
load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
|
||||||
|
|
||||||
|
case "
|
||||||
|
$load_sops_seen
|
||||||
|
" in
|
||||||
|
*"
|
||||||
|
$load_sops_name
|
||||||
|
"*)
|
||||||
|
if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
|
||||||
|
printf 'load-sops: normalized environment name collision\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
load_sops_seen=${load_sops_seen}${load_sops_seen:+"
|
||||||
|
"}$load_sops_name
|
||||||
|
|
||||||
|
load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
|
||||||
|
printf 'load-sops: failed to inspect YAML value kind\n' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
|
||||||
|
printf 'load-sops: failed to inspect YAML value tag\n' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$load_sops_kind" != scalar ]; then
|
||||||
|
printf 'load-sops: top-level YAML values must be scalars\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_tag" = '!!null' ]; then
|
||||||
|
printf 'load-sops: top-level YAML null values are not supported\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
|
||||||
|
eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
|
||||||
|
if [ -n "$load_sops_already_set" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
|
||||||
|
printf 'load-sops: failed to read YAML scalar value\n' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
|
||||||
|
eval "export $load_sops_name=$load_sops_quoted"
|
||||||
|
done <<EOF
|
||||||
|
$load_sops_keys
|
||||||
|
EOF
|
||||||
|
}
|
||||||
@@ -9,11 +9,11 @@ with self.lib;
|
|||||||
let
|
let
|
||||||
# Combine hectic modules into one
|
# Combine hectic modules into one
|
||||||
hectic.imports = attrValues (
|
hectic.imports = attrValues (
|
||||||
readModulesRecursive' ./hectic { inherit flake self inputs; }
|
readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
|
||||||
);
|
);
|
||||||
# Read generic modules separately
|
# Read generic modules separately
|
||||||
generic = readModulesRecursive'
|
generic = readModulesRecursive'
|
||||||
./generic
|
(flake + "/nixos/module/generic")
|
||||||
{ inherit flake self inputs; };
|
{ inherit flake self inputs; };
|
||||||
in generic // {
|
in generic // {
|
||||||
inherit hectic;
|
inherit hectic;
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
];
|
];
|
||||||
|
|
||||||
adminNames = [ "yukkop" ];
|
adminNames = [ "yukkop" ];
|
||||||
|
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||||
|
|
||||||
cfg = config.hectic.generic.matrix-cluster;
|
cfg = config.hectic.generic.matrix-cluster;
|
||||||
in {
|
in {
|
||||||
@@ -39,7 +40,7 @@ in {
|
|||||||
value = {
|
value = {
|
||||||
key = "matrix/users/${name}/password";
|
key = "matrix/users/${name}/password";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
}) userNames
|
}) userNames
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -250,6 +250,7 @@ in {
|
|||||||
# failover flip does not need a separate provisioning step.
|
# failover flip does not need a separate provisioning step.
|
||||||
systemd.tmpfiles.rules = [
|
systemd.tmpfiles.rules = [
|
||||||
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
|
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
|
||||||
|
"Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
|
||||||
];
|
];
|
||||||
|
|
||||||
systemd.services.matrix-cluster-signing-key = {
|
systemd.services.matrix-cluster-signing-key = {
|
||||||
@@ -319,6 +320,12 @@ in {
|
|||||||
media_store_path = s3Cfg.mediaStorePath;
|
media_store_path = s3Cfg.mediaStorePath;
|
||||||
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
|
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
|
||||||
|
|
||||||
|
# Tolerate bursty Element/iPhone presence syncs without disabling limits.
|
||||||
|
rc_presence.per_user = {
|
||||||
|
per_second = 0.5;
|
||||||
|
burst_count = 5;
|
||||||
|
};
|
||||||
|
|
||||||
experimental_features = {
|
experimental_features = {
|
||||||
msc3266_enabled = true;
|
msc3266_enabled = true;
|
||||||
msc4140_enabled = true;
|
msc4140_enabled = true;
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
{
|
{
|
||||||
inputs,
|
...
|
||||||
flake,
|
|
||||||
self,
|
|
||||||
}:
|
}:
|
||||||
{
|
{
|
||||||
pkgs,
|
pkgs,
|
||||||
@@ -11,6 +9,13 @@
|
|||||||
}: let
|
}: let
|
||||||
cfg = config.hectic.hardware.hetzner-cloud;
|
cfg = config.hectic.hardware.hetzner-cloud;
|
||||||
isNewer = cfg.generation == "newer";
|
isNewer = cfg.generation == "newer";
|
||||||
|
networkMatchConfig =
|
||||||
|
(lib.optionalAttrs (cfg.networkMatchConfigName != null) {
|
||||||
|
Name = cfg.networkMatchConfigName;
|
||||||
|
})
|
||||||
|
// (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
|
||||||
|
PermanentMACAddress = cfg.networkMatchConfigMac;
|
||||||
|
});
|
||||||
in {
|
in {
|
||||||
options.hectic.hardware.hetzner-cloud = {
|
options.hectic.hardware.hetzner-cloud = {
|
||||||
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
|
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
|
||||||
@@ -51,6 +56,14 @@ in {
|
|||||||
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
floatingIpv4 = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
|
||||||
|
default = null;
|
||||||
|
example = "188.243.124.247";
|
||||||
|
description = ''
|
||||||
|
Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
|
||||||
|
'';
|
||||||
|
};
|
||||||
device = lib.mkOption {
|
device = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
|
default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
|
||||||
@@ -65,14 +78,27 @@ in {
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
networkMatchConfigName = lib.mkOption {
|
networkMatchConfigName = lib.mkOption {
|
||||||
type = lib.types.strMatching "^(enp1s0|ens3|eth0)$";
|
type = with lib.types; nullOr str;
|
||||||
|
default = null;
|
||||||
example = "enp1s0";
|
example = "enp1s0";
|
||||||
description = ''
|
description = ''
|
||||||
type of network conection,
|
Optional interface name to match in systemd-networkd.
|
||||||
on older hetzner servers may be `ens3` or else
|
|
||||||
on newer probably `enp1s0`
|
|
||||||
|
|
||||||
you can use `networkctl list` on server to know it
|
Prefer `networkMatchConfigMac` for stable matching across rescue
|
||||||
|
images and installed systems that may rename interfaces differently.
|
||||||
|
|
||||||
|
You can use `networkctl list` on server to know it.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networkMatchConfigMac = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
|
||||||
|
default = null;
|
||||||
|
example = "92:00:08:4a:b0:32";
|
||||||
|
description = ''
|
||||||
|
Optional permanent MAC address to match in systemd-networkd.
|
||||||
|
|
||||||
|
This is the preferred Hetzner Cloud matching method because interface
|
||||||
|
names can differ between rescue images and installed NixOS systems.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -80,6 +106,15 @@ in {
|
|||||||
config = lib.mkIf cfg.enable (lib.mkMerge
|
config = lib.mkIf cfg.enable (lib.mkMerge
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
|
boot.loader.systemd-boot.enable = false;
|
||||||
|
boot.loader.efi.canTouchEfiVariables = false;
|
||||||
|
boot.loader.grub = {
|
||||||
|
enable = true;
|
||||||
|
efiSupport = true;
|
||||||
|
efiInstallAsRemovable = true;
|
||||||
|
device = "nodev";
|
||||||
|
};
|
||||||
|
|
||||||
boot.initrd.availableKernelModules = [
|
boot.initrd.availableKernelModules = [
|
||||||
"ata_piix"
|
"ata_piix"
|
||||||
"uhci_hcd"
|
"uhci_hcd"
|
||||||
@@ -90,12 +125,12 @@ in {
|
|||||||
networking.useNetworkd = true;
|
networking.useNetworkd = true;
|
||||||
systemd.network.enable = true;
|
systemd.network.enable = true;
|
||||||
systemd.network.networks."30-wan" = {
|
systemd.network.networks."30-wan" = {
|
||||||
matchConfig.Name = cfg.networkMatchConfigName;
|
matchConfig = networkMatchConfig;
|
||||||
networkConfig.DHCP = "no";
|
networkConfig.DHCP = "no";
|
||||||
address = [
|
address = [
|
||||||
"${cfg.ipv4}/32"
|
"${cfg.ipv4}/32"
|
||||||
"${cfg.ipv6}::/64"
|
"${cfg.ipv6}::/64"
|
||||||
];
|
] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
|
||||||
routes = [
|
routes = [
|
||||||
{ Gateway = "172.31.1.1"; GatewayOnLink = true; }
|
{ Gateway = "172.31.1.1"; GatewayOnLink = true; }
|
||||||
{ Gateway = "fe80::1"; }
|
{ Gateway = "fe80::1"; }
|
||||||
@@ -137,6 +172,13 @@ in {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
|
||||||
|
message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
|
||||||
|
}
|
||||||
|
];
|
||||||
}
|
}
|
||||||
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
|
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
|
||||||
boot.initrd.kernelModules = [ "virtio_gpu" ];
|
boot.initrd.kernelModules = [ "virtio_gpu" ];
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
{ ... }:
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
cfg = config.hectic.hardware.njalla;
|
||||||
|
in {
|
||||||
|
options.hectic.hardware.njalla = {
|
||||||
|
enable = lib.mkEnableOption "Enable njalla hardware configurations";
|
||||||
|
ipv4 = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
|
||||||
|
example = "185.193.126.103";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv4 address assigned to the host.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv4PrefixLength = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
default = 24;
|
||||||
|
example = 24;
|
||||||
|
description = ''
|
||||||
|
Njalla IPv4 prefix length.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv4Gateway = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
|
||||||
|
default = "185.193.126.1";
|
||||||
|
example = "185.193.126.1";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv4 gateway.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv6 = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
|
||||||
|
example = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv6 address assigned to the host.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv6PrefixLength = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
default = 64;
|
||||||
|
example = 64;
|
||||||
|
description = ''
|
||||||
|
Njalla IPv6 prefix length.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv6Gateway = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
|
||||||
|
default = "2a0a:3840:1337:126::1";
|
||||||
|
example = "2a0a:3840:1337:126::1";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv6 gateway.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networkMatchConfigName = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "eth0";
|
||||||
|
example = "eth0";
|
||||||
|
description = ''
|
||||||
|
Njalla container network interface name.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
device = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "/dev/vda";
|
||||||
|
example = "/dev/disk/by-id/virtio-root";
|
||||||
|
description = ''
|
||||||
|
Njalla installation disk for disko/nixos-anywhere.
|
||||||
|
|
||||||
|
`/dev/vda` is the default block device visible on the inspected Njalla
|
||||||
|
host. Prefer a stable `/dev/disk/by-id/...` path when available.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
enableDisko = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = ''
|
||||||
|
Whether to provide a disko layout for nixos-anywhere installs.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable (lib.mkMerge [
|
||||||
|
{
|
||||||
|
boot.isContainer = true;
|
||||||
|
|
||||||
|
networking.useDHCP = false;
|
||||||
|
networking.useNetworkd = true;
|
||||||
|
systemd.network.enable = true;
|
||||||
|
systemd.network.networks."30-wan" = {
|
||||||
|
matchConfig.Name = cfg.networkMatchConfigName;
|
||||||
|
networkConfig.DHCP = "no";
|
||||||
|
address = [
|
||||||
|
"${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
|
||||||
|
"${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
|
||||||
|
];
|
||||||
|
routes = [
|
||||||
|
{ Gateway = cfg.ipv4Gateway; }
|
||||||
|
{ Gateway = cfg.ipv6Gateway; }
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
|
||||||
|
}
|
||||||
|
(lib.mkIf cfg.enableDisko {
|
||||||
|
boot.loader.grub.device = cfg.device;
|
||||||
|
|
||||||
|
disko.devices.disk.main = {
|
||||||
|
type = "disk";
|
||||||
|
device = cfg.device;
|
||||||
|
content = {
|
||||||
|
type = "gpt";
|
||||||
|
partitions = {
|
||||||
|
boot = {
|
||||||
|
size = "1M";
|
||||||
|
type = "EF02";
|
||||||
|
priority = 1;
|
||||||
|
};
|
||||||
|
root = {
|
||||||
|
size = "100%";
|
||||||
|
content = {
|
||||||
|
type = "filesystem";
|
||||||
|
format = "ext4";
|
||||||
|
mountpoint = "/";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
}
|
||||||
@@ -25,36 +25,7 @@ in {
|
|||||||
programs.bash.shellAliases.tmux = "tmux a";
|
programs.bash.shellAliases.tmux = "tmux a";
|
||||||
|
|
||||||
home-manager.sharedModules = [
|
home-manager.sharedModules = [
|
||||||
{
|
(flake + "/home/module/program/tmux.nix")
|
||||||
programs.tmux = {
|
|
||||||
enable = true;
|
|
||||||
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
|
|
||||||
keyMode = "vi";
|
|
||||||
escapeTime = 500;
|
|
||||||
historyLimit = 50000;
|
|
||||||
newSession = true;
|
|
||||||
extraConfig = ''
|
|
||||||
# resurrect
|
|
||||||
set -g @resurrect-strategy-vim 'session'
|
|
||||||
set -g @resurrect-strategy-nvim 'session'
|
|
||||||
set -g @resurrect-capture-pane-contents 'on'
|
|
||||||
|
|
||||||
resurrect_dir="$HOME/.tmux/resurrect"
|
|
||||||
set -g @resurrect-dir $resurrect_dir
|
|
||||||
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
|
|
||||||
|
|
||||||
# continuum
|
|
||||||
set -g @continuum-restore 'on'
|
|
||||||
set -g @continuum-boot 'on'
|
|
||||||
set -g @continuum-save-interval '10'
|
|
||||||
|
|
||||||
bind-key -T copy-mode-vi v send-keys -X begin-selection
|
|
||||||
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
|
|
||||||
|
|
||||||
bind-key O select-pane -t :.-
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
];
|
||||||
|
|
||||||
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
|
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
|
||||||
|
|||||||
@@ -147,7 +147,7 @@ in {
|
|||||||
|
|
||||||
(lib.mkIf cfg.watcher.enable {
|
(lib.mkIf cfg.watcher.enable {
|
||||||
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
|
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
|
||||||
sopsFile = "${flake}/sus/sentinella-default.yaml";
|
sopsFile = flake + "/sus/sentinella-default.yaml";
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.services."sentinella-watcher" = {
|
systemd.services."sentinella-watcher" = {
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
{ ... }: {
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
cfg = config.hectic.services.stable-video-diffusion;
|
||||||
|
in {
|
||||||
|
options.hectic.services.stable-video-diffusion = {
|
||||||
|
enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
|
||||||
|
|
||||||
|
host = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "127.0.0.1";
|
||||||
|
description = "Address the Stable Video Diffusion API binds to.";
|
||||||
|
};
|
||||||
|
|
||||||
|
port = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 7861;
|
||||||
|
description = "Port the Stable Video Diffusion API binds to.";
|
||||||
|
};
|
||||||
|
|
||||||
|
package = lib.mkOption {
|
||||||
|
type = lib.types.package;
|
||||||
|
default = pkgs.hectic.stable-video-diffusion-api;
|
||||||
|
defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
|
||||||
|
description = "Package providing the Stable Video Diffusion API executable.";
|
||||||
|
};
|
||||||
|
|
||||||
|
modelId = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "stabilityai/stable-video-diffusion-img2vid-xt";
|
||||||
|
description = "Model identifier loaded by the Stable Video Diffusion API.";
|
||||||
|
};
|
||||||
|
|
||||||
|
device = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
|
||||||
|
default = null;
|
||||||
|
description = ''
|
||||||
|
Torch device requested from the Stable Video Diffusion API. When null,
|
||||||
|
the package keeps its own auto-detection behavior.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
libraryPath = lib.mkOption {
|
||||||
|
type = with lib.types; listOf str;
|
||||||
|
default = [];
|
||||||
|
description = ''
|
||||||
|
Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
|
||||||
|
should include /run/opengl-driver/lib so libcuda.so is visible to torch.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
stateDir = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "/var/lib/stable-video-diffusion-api";
|
||||||
|
description = "Persistent state directory for the Stable Video Diffusion API.";
|
||||||
|
};
|
||||||
|
|
||||||
|
cacheDir = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "/var/cache/stable-video-diffusion-api";
|
||||||
|
description = "Cache directory for downloaded model and runtime artifacts.";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputDir = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "${cfg.stateDir}/outputs";
|
||||||
|
defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
|
||||||
|
description = "Directory where generated video outputs are written.";
|
||||||
|
};
|
||||||
|
|
||||||
|
environmentFile = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr path;
|
||||||
|
default = null;
|
||||||
|
description = ''
|
||||||
|
Optional environment file for secrets or runtime overrides. Values from
|
||||||
|
the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
|
||||||
|
SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
|
||||||
|
and LD_LIBRARY_PATH by default.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
openFirewall = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Whether to open the API port in the firewall.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable {
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = cfg.device != "cuda" || cfg.libraryPath != [];
|
||||||
|
message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
users.users.stable-video-diffusion = {
|
||||||
|
isSystemUser = true;
|
||||||
|
group = "stable-video-diffusion";
|
||||||
|
};
|
||||||
|
users.groups.stable-video-diffusion = {};
|
||||||
|
|
||||||
|
systemd.tmpfiles.rules = [
|
||||||
|
"d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||||
|
"d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||||
|
"d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||||
|
];
|
||||||
|
|
||||||
|
systemd.services.stable-video-diffusion-api = {
|
||||||
|
description = "Stable Video Diffusion HTTP API";
|
||||||
|
after = [ "network.target" ];
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
environment = {
|
||||||
|
SVD_API_HOST = cfg.host;
|
||||||
|
SVD_API_PORT = toString cfg.port;
|
||||||
|
SVD_MODEL_ID = cfg.modelId;
|
||||||
|
SVD_STATE_DIR = cfg.stateDir;
|
||||||
|
SVD_CACHE_DIR = cfg.cacheDir;
|
||||||
|
SVD_OUTPUT_DIR = cfg.outputDir;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.device != null) {
|
||||||
|
SVD_DEVICE = cfg.device;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.libraryPath != []) {
|
||||||
|
LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
|
||||||
|
};
|
||||||
|
serviceConfig = lib.mkMerge [
|
||||||
|
{
|
||||||
|
Type = "simple";
|
||||||
|
User = "stable-video-diffusion";
|
||||||
|
Group = "stable-video-diffusion";
|
||||||
|
WorkingDirectory = cfg.stateDir;
|
||||||
|
ExecStart = lib.getExe' cfg.package "stable-video-diffusion-api";
|
||||||
|
Restart = "on-failure";
|
||||||
|
RestartSec = "5s";
|
||||||
|
TimeoutStopSec = "30s";
|
||||||
|
KillSignal = "SIGTERM";
|
||||||
|
KillMode = "mixed";
|
||||||
|
StandardOutput = "journal";
|
||||||
|
StandardError = "journal";
|
||||||
|
}
|
||||||
|
(lib.mkIf (cfg.environmentFile != null) {
|
||||||
|
EnvironmentFile = cfg.environmentFile;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,230 +0,0 @@
|
|||||||
# Matrix Cluster Failover Runbook (`accord.tube`)
|
|
||||||
|
|
||||||
Primary: `hectic-lab` (NL, `128.140.75.58`)
|
|
||||||
Standby: `bfs.poland.xray` (PL, `91.198.166.181`)
|
|
||||||
|
|
||||||
Module: `hectic.generic.matrix-cluster` (`nixos/module/generic/matrix-cluster.nix`).
|
|
||||||
Shared secrets: `sus/matrix-cluster.yaml`.
|
|
||||||
|
|
||||||
All `psql` and `pg_ctl` invocations use PostgreSQL **17** at data dir
|
|
||||||
`/var/lib/postgresql/17`.
|
|
||||||
|
|
||||||
## Initial setup
|
|
||||||
|
|
||||||
### 1. Provision shared SOPS file (`sus/matrix-cluster.yaml`)
|
|
||||||
|
|
||||||
On a workstation with both yukkop and yukkop-alt age keys available:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo cat /var/lib/matrix-synapse/homeserver.signing.key # on NL (hectic-lab)
|
|
||||||
# Copy the single line value into the buffer for the next step.
|
|
||||||
|
|
||||||
sops sus/matrix-cluster.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
Populate the editor with:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
matrix:
|
|
||||||
signing-key: <paste verbatim signing-key line from NL>
|
|
||||||
postgres-replication-password: <openssl rand -base64 32>
|
|
||||||
object-storage:
|
|
||||||
credentials: |
|
|
||||||
ACCESS_KEY_ID=<verbatim copy from sus/hectic-lab.yaml>
|
|
||||||
SECRET_ACCESS_KEY=<verbatim copy from sus/hectic-lab.yaml>
|
|
||||||
porkbun-api-key: <PORKBUN_API_KEY>
|
|
||||||
porkbun-secret-api-key: <PORKBUN_SECRET_API_KEY>
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify recipients:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sops updatekeys sus/matrix-cluster.yaml
|
|
||||||
sops -d sus/matrix-cluster.yaml | grep -E 'signing-key|porkbun-api-key|object-storage'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: all five keys present, exit 0.
|
|
||||||
|
|
||||||
### 2. Deploy NL primary first
|
|
||||||
|
|
||||||
```sh
|
|
||||||
nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux' --target-host root@128.140.75.58
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify on NL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo systemctl status matrix-synapse postgresql matrix-cluster-replication-password
|
|
||||||
sudo -u postgres psql -c "select rolname, rolreplication from pg_roles where rolname='replication';"
|
|
||||||
# Expected: replication | t
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Seed PL replica with `pg_basebackup`
|
|
||||||
|
|
||||||
On PL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo systemctl stop postgresql
|
|
||||||
sudo rm -rf /var/lib/postgresql/17
|
|
||||||
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
|
|
||||||
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
|
|
||||||
pg_basebackup \
|
|
||||||
-h 128.140.75.58 \
|
|
||||||
-p 5432 \
|
|
||||||
-U replication \
|
|
||||||
-D /var/lib/postgresql/17 \
|
|
||||||
-Fp -Xs -P -R \
|
|
||||||
--no-password
|
|
||||||
```
|
|
||||||
|
|
||||||
`-R` writes `standby.signal` and an initial `primary_conninfo`. The
|
|
||||||
matrix-cluster module's `matrix-cluster-standby-bootstrap` service will
|
|
||||||
overwrite `primary_conninfo` to use a libpq passfile on next boot.
|
|
||||||
|
|
||||||
### 4. Deploy PL standby
|
|
||||||
|
|
||||||
```sh
|
|
||||||
nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux' --target-host root@91.198.166.181
|
|
||||||
sudo systemctl start postgresql
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify streaming on NL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo -u postgres psql -c 'select client_addr, state, sync_state from pg_stat_replication;'
|
|
||||||
# Expected: 91.198.166.181 | streaming | async
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify standby on PL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo -u postgres psql -c 'select pg_is_in_recovery();'
|
|
||||||
# Expected: t
|
|
||||||
sudo systemctl is-active matrix-synapse
|
|
||||||
# Expected: inactive (standby keeps Synapse off)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Remove duplicate S3 credentials from `sus/hectic-lab.yaml`
|
|
||||||
|
|
||||||
Only AFTER NL is confirmed healthy reading from the new shared file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sops sus/hectic-lab.yaml
|
|
||||||
# Delete the matrix/object-storage/credentials block.
|
|
||||||
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
|
|
||||||
```
|
|
||||||
|
|
||||||
## Normal operations
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# NL: replication health
|
|
||||||
sudo -u postgres psql -c 'select * from pg_stat_replication;'
|
|
||||||
# Expected: 1 row, state=streaming, sync_state=async
|
|
||||||
|
|
||||||
# PL: replay status
|
|
||||||
sudo -u postgres psql -c 'select now() - pg_last_xact_replay_timestamp() as lag;'
|
|
||||||
|
|
||||||
# Both: cert renewal
|
|
||||||
sudo systemctl status acme-accord.tube.timer
|
|
||||||
sudo journalctl -u acme-accord.tube.service --since '24 hours ago'
|
|
||||||
|
|
||||||
# Synapse health (NL primary)
|
|
||||||
curl -sf https://accord.tube/_matrix/client/versions | head
|
|
||||||
```
|
|
||||||
|
|
||||||
## Planned failover (NL -> PL)
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# 1. Drain NL: stop accepting writes.
|
|
||||||
sudo systemctl stop matrix-synapse
|
|
||||||
sudo systemctl stop postgresql # ensure no new WAL after this point
|
|
||||||
|
|
||||||
# 2. Promote PL replica.
|
|
||||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
|
|
||||||
# Wait until pg_is_in_recovery() returns f:
|
|
||||||
sudo -u postgres psql -c 'select pg_is_in_recovery();'
|
|
||||||
|
|
||||||
# 3. Make the role switch declarative before rebuilding.
|
|
||||||
# Edit the flake so rebuilds match the promoted database state:
|
|
||||||
# - nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
|
|
||||||
# hectic.generic.matrix-cluster.role = "primary";
|
|
||||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = true;
|
|
||||||
# hectic.generic.matrix-cluster.secretsFile = config.sops.secrets."matrix/secrets".path;
|
|
||||||
# - nixos/system/hectic-lab/hectic-lab.nix:
|
|
||||||
# hectic.generic.matrix-cluster.role = "standby";
|
|
||||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = false;
|
|
||||||
# hectic.generic.matrix-cluster.replication.peerHost = "91.198.166.181";
|
|
||||||
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "128.140.75.58/32" ];
|
|
||||||
# (You will also need a matrix/secrets entry on PL - copy from NL via SOPS.)
|
|
||||||
sudo nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux'
|
|
||||||
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
|
|
||||||
sudo systemctl status matrix-synapse
|
|
||||||
|
|
||||||
# 4. Swap DNS A record at Porkbun:
|
|
||||||
# accord.tube A 91.198.166.181 (was 128.140.75.58)
|
|
||||||
# TTL: set to 300 in advance of any planned failover.
|
|
||||||
# Porkbun UI: https://porkbun.com/account/domainsSpeedy -> accord.tube -> DNS -> edit A record.
|
|
||||||
# Or via API:
|
|
||||||
sudo curl -sX POST https://api.porkbun.com/api/json/v3/dns/editByNameType/accord.tube/A \
|
|
||||||
-H 'content-type: application/json' \
|
|
||||||
-d "$(jq -n --arg k "$PORKBUN_API_KEY" --arg s "$PORKBUN_SECRET_API_KEY" \
|
|
||||||
'{secretapikey:$s,apikey:$k,content:"91.198.166.181",ttl:"300"}')"
|
|
||||||
|
|
||||||
# 5. Federation smoke test.
|
|
||||||
curl -s 'https://federationtester.matrix.org/api/report?server_name=accord.tube' | jq .FederationOK
|
|
||||||
# Expected: true
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected after the rebuilds:
|
|
||||||
|
|
||||||
- `bfs.poland.xray` evaluates and runs as `role = "primary"`.
|
|
||||||
- `hectic-lab` evaluates as `role = "standby"` with Synapse forced off.
|
|
||||||
- Future `nixos-rebuild` runs preserve the promoted topology instead of reapplying standby settings to PL.
|
|
||||||
|
|
||||||
## Failback (PL -> NL)
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# 1. Stop NL postgres if still up; clear its data dir.
|
|
||||||
sudo systemctl stop postgresql matrix-synapse
|
|
||||||
sudo rm -rf /var/lib/postgresql/17
|
|
||||||
|
|
||||||
# 2. Re-seed NL from PL (now the live primary).
|
|
||||||
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
|
|
||||||
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
|
|
||||||
pg_basebackup -h 91.198.166.181 -p 5432 -U replication \
|
|
||||||
-D /var/lib/postgresql/17 -Fp -Xs -P -R --no-password
|
|
||||||
|
|
||||||
# 3. Temporarily flip roles in the flake:
|
|
||||||
# - hectic-lab.nix: role = "standby"; peerHost = "91.198.166.181";
|
|
||||||
# - bfs.poland.xray.nix: role = "primary"; peerHost = "128.140.75.58";
|
|
||||||
# Rebuild both.
|
|
||||||
|
|
||||||
# 4. Once NL is streaming green, do the reverse failover dance:
|
|
||||||
sudo systemctl stop matrix-synapse # on PL
|
|
||||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote # on NL
|
|
||||||
# Then revert the flake role assignments back to NL=primary / PL=standby and
|
|
||||||
# rebuild both hosts.
|
|
||||||
|
|
||||||
# 5. Swap DNS back at Porkbun (A -> 128.140.75.58).
|
|
||||||
```
|
|
||||||
|
|
||||||
## Disaster recovery (NL permanently lost)
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# 1. Promote PL as the new permanent primary.
|
|
||||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
|
|
||||||
|
|
||||||
# 2. Edit nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
|
|
||||||
# hectic.generic.matrix-cluster.role = "primary";
|
|
||||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = lib.mkForce null;
|
|
||||||
# hectic.generic.matrix-cluster.replication.peerHost = "<new-standby-ip>";
|
|
||||||
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "<new-standby-ip>/32" ];
|
|
||||||
|
|
||||||
# 3. Provision a new host (any region with Porkbun-managed DNS) and import
|
|
||||||
# self.nixosModules.matrix-cluster with role = "standby" pointed at PL's IP.
|
|
||||||
|
|
||||||
# 4. Bootstrap the new standby via pg_basebackup from PL exactly as in
|
|
||||||
# "Initial setup" step 3, replacing 128.140.75.58 with PL's IP.
|
|
||||||
|
|
||||||
# 5. Update Porkbun A record to PL's IP permanently.
|
|
||||||
```
|
|
||||||
@@ -9,16 +9,11 @@
|
|||||||
config,
|
config,
|
||||||
...
|
...
|
||||||
}: let
|
}: let
|
||||||
matrixBackend = "https://128.140.75.58";
|
|
||||||
matrixHost = "accord.tube";
|
matrixHost = "accord.tube";
|
||||||
jitsiHost = "meet.accord.tube";
|
jitsiHost = "meet.accord.tube";
|
||||||
elementEntryDomain = "element.bfs.band";
|
elementEntryDomain = "element.bfs.band";
|
||||||
polandEntryDomain = "bfs.band";
|
polandEntryDomain = "bfs.band";
|
||||||
backendProxyConfig = ''
|
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||||
proxy_ssl_server_name on;
|
|
||||||
proxy_ssl_name ${matrixHost};
|
|
||||||
proxy_set_header Host ${matrixHost};
|
|
||||||
'';
|
|
||||||
in {
|
in {
|
||||||
imports = [
|
imports = [
|
||||||
self.nixosModules.xray-system
|
self.nixosModules.xray-system
|
||||||
@@ -46,7 +41,6 @@ in {
|
|||||||
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
|
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
|
||||||
};
|
};
|
||||||
replication = {
|
replication = {
|
||||||
peerHost = "128.140.75.58";
|
|
||||||
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
|
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
|
||||||
};
|
};
|
||||||
acme = {
|
acme = {
|
||||||
@@ -71,6 +65,14 @@ in {
|
|||||||
hostName = jitsiHost;
|
hostName = jitsiHost;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# NOTE(yukkop): disk was provisioned outside disko, so the expected partition
|
||||||
|
# label does not exist. Pin root to the live filesystem UUID so stage 1 can
|
||||||
|
# mount `/` reliably.
|
||||||
|
fileSystems."/" = lib.mkForce {
|
||||||
|
device = "/dev/disk/by-uuid/06b48ef1-a1eb-428d-821c-90c96a624542";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
security.acme = {
|
security.acme = {
|
||||||
acceptTerms = true;
|
acceptTerms = true;
|
||||||
defaults.email = "security@bfs.band";
|
defaults.email = "security@bfs.band";
|
||||||
@@ -124,19 +126,17 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
locations."= /livekit/jwt" = {
|
locations."= /livekit/jwt" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/jwt";
|
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
|
||||||
extraConfig = backendProxyConfig;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /livekit/jwt/" = {
|
locations."^~ /livekit/jwt/" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/jwt/";
|
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
|
||||||
extraConfig = backendProxyConfig;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
locations."= /livekit/sfu" = {
|
locations."= /livekit/sfu" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/sfu";
|
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
|
||||||
proxyWebsockets = true;
|
proxyWebsockets = true;
|
||||||
extraConfig = backendProxyConfig + ''
|
extraConfig = ''
|
||||||
proxy_send_timeout 120;
|
proxy_send_timeout 120;
|
||||||
proxy_read_timeout 120;
|
proxy_read_timeout 120;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
@@ -147,9 +147,9 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /livekit/sfu/" = {
|
locations."^~ /livekit/sfu/" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/sfu/";
|
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
|
||||||
proxyWebsockets = true;
|
proxyWebsockets = true;
|
||||||
extraConfig = backendProxyConfig + ''
|
extraConfig = ''
|
||||||
proxy_send_timeout 120;
|
proxy_send_timeout 120;
|
||||||
proxy_read_timeout 120;
|
proxy_read_timeout 120;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
@@ -160,13 +160,14 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /_matrix/" = {
|
locations."^~ /_matrix/" = {
|
||||||
proxyPass = "${matrixBackend}/_matrix/";
|
proxyPass = "http://127.0.0.1:8008";
|
||||||
extraConfig = backendProxyConfig;
|
extraConfig = ''
|
||||||
|
client_max_body_size ${config.hectic.generic.matrix-cluster.maxUploadSize};
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /_synapse/client/" = {
|
locations."^~ /_synapse/client/" = {
|
||||||
proxyPass = "${matrixBackend}/_synapse/client/";
|
proxyPass = "http://127.0.0.1:8008";
|
||||||
extraConfig = backendProxyConfig;
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -203,41 +204,41 @@ in {
|
|||||||
key = "matrix/signing-key";
|
key = "matrix/signing-key";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/postgres-replication-password" = {
|
sops.secrets."matrix/postgres-replication-password" = {
|
||||||
key = "matrix/postgres-replication-password";
|
key = "matrix/postgres-replication-password";
|
||||||
owner = "postgres";
|
owner = "postgres";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/object-storage/credentials" = {
|
sops.secrets."matrix/object-storage/credentials" = {
|
||||||
key = "matrix/object-storage/credentials";
|
key = "matrix/object-storage/credentials";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/secrets" = {
|
sops.secrets."matrix/secrets" = {
|
||||||
key = "matrix/secrets";
|
key = "matrix/secrets";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/turn-secret" = {
|
sops.secrets."matrix/turn-secret" = {
|
||||||
key = "matrix/turn-secret";
|
key = "matrix/turn-secret";
|
||||||
owner = "turnserver";
|
owner = "turnserver";
|
||||||
group = "turnserver";
|
group = "turnserver";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/porkbun-api-key" = {
|
sops.secrets."matrix/porkbun-api-key" = {
|
||||||
key = "matrix/porkbun-api-key";
|
key = "matrix/porkbun-api-key";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/porkbun-secret-api-key" = {
|
sops.secrets."matrix/porkbun-secret-api-key" = {
|
||||||
key = "matrix/porkbun-secret-api-key";
|
key = "matrix/porkbun-secret-api-key";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
{ pkgs, ... }:
|
||||||
|
let
|
||||||
|
port = 22222;
|
||||||
|
stateDir = "/var/lib/experimental-sshd";
|
||||||
|
configFile = pkgs.writeText "experimental-sshd_config" ''
|
||||||
|
Port 22222
|
||||||
|
ListenAddress 0.0.0.0
|
||||||
|
ListenAddress ::
|
||||||
|
|
||||||
|
HostKey ${stateDir}/ssh_host_ed25519_key
|
||||||
|
HostKey ${stateDir}/ssh_host_rsa_key
|
||||||
|
|
||||||
|
PasswordAuthentication no
|
||||||
|
KbdInteractiveAuthentication no
|
||||||
|
PubkeyAuthentication yes
|
||||||
|
PermitRootLogin prohibit-password
|
||||||
|
UsePAM yes
|
||||||
|
AuthenticationMethods publickey
|
||||||
|
AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u
|
||||||
|
LogLevel DEBUG3
|
||||||
|
|
||||||
|
VersionAddendum none
|
||||||
|
HostKeyAlgorithms rsa-sha2-512,rsa-sha2-256,ssh-ed25519
|
||||||
|
KexAlgorithms curve25519-sha256,diffie-hellman-group14-sha256
|
||||||
|
Ciphers aes256-ctr,aes128-ctr
|
||||||
|
MACs hmac-sha2-512,hmac-sha2-256
|
||||||
|
'';
|
||||||
|
|
||||||
|
keygenScript = pkgs.writeShellScript "experimental-sshd-keygen" ''
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
${pkgs.coreutils}/bin/mkdir -p -m 0700 ${stateDir}
|
||||||
|
|
||||||
|
if [ ! -f ${stateDir}/ssh_host_ed25519_key ]; then
|
||||||
|
${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f ${stateDir}/ssh_host_ed25519_key -N ""
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f ${stateDir}/ssh_host_rsa_key ]; then
|
||||||
|
${pkgs.openssh}/bin/ssh-keygen -t rsa -b 4096 -f ${stateDir}/ssh_host_rsa_key -N ""
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
environment.etc."ssh/experimental-sshd_config".source = configFile;
|
||||||
|
|
||||||
|
networking.firewall.allowedTCPPorts = [ port ];
|
||||||
|
|
||||||
|
systemd.services.experimental-sshd-keygen = {
|
||||||
|
description = "Generate experimental SSH host keys";
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
StateDirectory = "experimental-sshd";
|
||||||
|
ExecStart = keygenScript;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services.experimental-sshd = {
|
||||||
|
description = "Experimental SSH daemon";
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
wants = [ "experimental-sshd-keygen.service" ];
|
||||||
|
after = [ "network.target" "experimental-sshd-keygen.service" ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "simple";
|
||||||
|
StateDirectory = "experimental-sshd";
|
||||||
|
RuntimeDirectory = "experimental-sshd";
|
||||||
|
ExecStart = "${pkgs.openssh}/bin/sshd -D -e -f /etc/ssh/experimental-sshd_config";
|
||||||
|
};
|
||||||
|
preStart = ''
|
||||||
|
${pkgs.openssh}/bin/sshd -t -f /etc/ssh/experimental-sshd_config
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -15,7 +15,7 @@ with builtins;
|
|||||||
with lib;
|
with lib;
|
||||||
let
|
let
|
||||||
domain = "hectic-lab.com";
|
domain = "hectic-lab.com";
|
||||||
matrixDomain = "accord.tube";
|
sshPort = 22;
|
||||||
mailUserNames = [
|
mailUserNames = [
|
||||||
"security"
|
"security"
|
||||||
"founders"
|
"founders"
|
||||||
@@ -57,6 +57,7 @@ in {
|
|||||||
|
|
||||||
(import ./attic.nix { inherit flake self inputs domain; })
|
(import ./attic.nix { inherit flake self inputs domain; })
|
||||||
(import ./containers.nix { inherit flake self inputs; })
|
(import ./containers.nix { inherit flake self inputs; })
|
||||||
|
./experimental-sshd.nix
|
||||||
(import ./ente.nix { inherit domain; })
|
(import ./ente.nix { inherit domain; })
|
||||||
(import ./mechabellum.nix { inherit flake self inputs domain; })
|
(import ./mechabellum.nix { inherit flake self inputs domain; })
|
||||||
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
|
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
|
||||||
@@ -80,6 +81,7 @@ in {
|
|||||||
enable = true;
|
enable = true;
|
||||||
networkMatchConfigName = "enp1s0";
|
networkMatchConfigName = "enp1s0";
|
||||||
ipv4 = "128.140.75.58";
|
ipv4 = "128.140.75.58";
|
||||||
|
floatingIpv4 = "78.47.243.0";
|
||||||
ipv6 = "2a01:4f8:c2c:d54a";
|
ipv6 = "2a01:4f8:c2c:d54a";
|
||||||
};
|
};
|
||||||
services.matrix = {
|
services.matrix = {
|
||||||
@@ -106,6 +108,7 @@ in {
|
|||||||
'';
|
'';
|
||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
|
tcpdump
|
||||||
git
|
git
|
||||||
rsync
|
rsync
|
||||||
python311
|
python311
|
||||||
@@ -116,7 +119,7 @@ in {
|
|||||||
sops = {
|
sops = {
|
||||||
gnupg.sshKeyPaths = [ ];
|
gnupg.sshKeyPaths = [ ];
|
||||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||||
defaultSopsFile = "${flake}/sus/hectic-lab.yaml";
|
defaultSopsFile = flake + "/sus/hectic-lab.yaml";
|
||||||
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
|
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
|
||||||
"init-postgresql" = {
|
"init-postgresql" = {
|
||||||
key = "init-postgresql";
|
key = "init-postgresql";
|
||||||
@@ -133,6 +136,8 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
users.users.root.openssh.authorizedKeys.keys = [
|
users.users.root.openssh.authorizedKeys.keys = [
|
||||||
|
# neuro machine
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
|
||||||
# yukkop
|
# yukkop
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
|
||||||
@@ -159,6 +164,8 @@ in {
|
|||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
services.openssh.ports = [ sshPort ];
|
||||||
|
|
||||||
services.mailserver = {
|
services.mailserver = {
|
||||||
enable = true;
|
enable = true;
|
||||||
domain = domain;
|
domain = domain;
|
||||||
@@ -179,10 +186,10 @@ in {
|
|||||||
|
|
||||||
networking.firewall = {
|
networking.firewall = {
|
||||||
allowedTCPPorts = [
|
allowedTCPPorts = [
|
||||||
|
sshPort # ssh
|
||||||
80
|
80
|
||||||
443
|
443
|
||||||
3306 # mysql
|
3306 # mysql
|
||||||
11012 # gitea ssh
|
|
||||||
25565
|
25565
|
||||||
55228 # ss-bfs
|
55228 # ss-bfs
|
||||||
];
|
];
|
||||||
@@ -264,7 +271,7 @@ in {
|
|||||||
settings.service.DISABLE_REGISTRATION = true;
|
settings.service.DISABLE_REGISTRATION = true;
|
||||||
settings.server = {
|
settings.server = {
|
||||||
HTTP_PORT = 11011;
|
HTTP_PORT = 11011;
|
||||||
#SSH_PORT = 22;
|
SSH_PORT = sshPort;
|
||||||
SSH_DOMAIN = "hectic-lab.com";
|
SSH_DOMAIN = "hectic-lab.com";
|
||||||
};
|
};
|
||||||
database = {
|
database = {
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ in {
|
|||||||
proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
|
proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
|
client_max_body_size 500M;
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -15,32 +15,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
|
|||||||
self.overlays.default
|
self.overlays.default
|
||||||
inputs.nix-minecraft.overlay
|
inputs.nix-minecraft.overlay
|
||||||
];
|
];
|
||||||
config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
config.allowUnfreePredicate = pkg:
|
||||||
|
self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
||||||
"minecraft-server"
|
"minecraft-server"
|
||||||
"neoforge"
|
"neoforge"
|
||||||
|
|
||||||
"nvidia-x11"
|
"nvidia-x11"
|
||||||
|
|
||||||
"cuda_nvcc"
|
|
||||||
"cuda_cudart"
|
|
||||||
"cuda_cuobjdump"
|
|
||||||
"cuda_cupti"
|
|
||||||
"cuda_nvdisasm"
|
|
||||||
"cuda_cccl"
|
|
||||||
"cuda_nvml_dev"
|
|
||||||
"cuda_nvrtc"
|
|
||||||
"cuda_nvtx"
|
|
||||||
"cuda_profiler_api"
|
|
||||||
|
|
||||||
"libcusparse_lt"
|
|
||||||
"libcublas"
|
|
||||||
"libcufft"
|
|
||||||
"libcufile"
|
|
||||||
"libcurand"
|
|
||||||
"libcusolver"
|
|
||||||
"libnvjitlink"
|
|
||||||
"libcusparse"
|
|
||||||
"cudnn"
|
|
||||||
];
|
];
|
||||||
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
|
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
|
||||||
config.permittedInsecurePackages = [
|
config.permittedInsecurePackages = [
|
||||||
|
|||||||
@@ -17,6 +17,11 @@
|
|||||||
|
|
||||||
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
|
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
|
||||||
|
|
||||||
|
stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
|
||||||
|
pkgs.stdenv.cc.cc.lib
|
||||||
|
pkgs.zlib
|
||||||
|
];
|
||||||
|
|
||||||
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
|
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
|
||||||
pname = "ollama";
|
pname = "ollama";
|
||||||
version = "0.22.1";
|
version = "0.22.1";
|
||||||
@@ -176,6 +181,19 @@ in {
|
|||||||
openFirewall = false;
|
openFirewall = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
hectic.services.stable-video-diffusion = {
|
||||||
|
enable = true;
|
||||||
|
host = "127.0.0.1";
|
||||||
|
port = 7861;
|
||||||
|
package = pkgs.hectic.stable-video-diffusion-api;
|
||||||
|
device = "cuda";
|
||||||
|
libraryPath = [
|
||||||
|
stableVideoDiffusionLibraryPath
|
||||||
|
"/run/opengl-driver/lib"
|
||||||
|
];
|
||||||
|
openFirewall = false;
|
||||||
|
};
|
||||||
|
|
||||||
networking = {
|
networking = {
|
||||||
networkmanager.enable = true;
|
networkmanager.enable = true;
|
||||||
useDHCP = lib.mkDefault true;
|
useDHCP = lib.mkDefault true;
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
flake,
|
||||||
|
self,
|
||||||
|
inputs,
|
||||||
|
system ? "x86_64-linux",
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
# Use folder name as name of this system
|
||||||
|
name = builtins.baseNameOf ./.;
|
||||||
|
|
||||||
|
in self.lib.nixpkgs-lib.nixosSystem {
|
||||||
|
pkgs = import inputs.nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
overlays = [ self.overlays.default ];
|
||||||
|
};
|
||||||
|
modules = [
|
||||||
|
{ networking.hostName = name; }
|
||||||
|
(import ./${name}.nix { inherit flake self inputs; })
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
{
|
||||||
|
inputs,
|
||||||
|
self,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
pkgs,
|
||||||
|
modulesPath,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
imports = [
|
||||||
|
(modulesPath + "/profiles/qemu-guest.nix")
|
||||||
|
inputs.sops-nix.nixosModules.sops
|
||||||
|
self.nixosModules.hectic
|
||||||
|
];
|
||||||
|
|
||||||
|
hectic = {
|
||||||
|
archetype.dev.enable = true;
|
||||||
|
hardware.hetzner-cloud = {
|
||||||
|
enable = true;
|
||||||
|
device = "/dev/sda";
|
||||||
|
networkMatchConfigMac = "92:00:08:4a:b0:32";
|
||||||
|
ipv4 = "46.225.237.218";
|
||||||
|
ipv6 = "2a01:4f8:c2c:3b14";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
users.users.root.openssh.authorizedKeys.keys = [
|
||||||
|
''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
|
||||||
|
];
|
||||||
|
|
||||||
|
environment.systemPackages = with pkgs; [
|
||||||
|
git
|
||||||
|
rsync
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -1,18 +1,21 @@
|
|||||||
# db-tool
|
# db-tool
|
||||||
|
|
||||||
PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries.
|
PostgreSQL utility package exposing separate development and operations binaries.
|
||||||
|
`db-dev` preserves the current development workflow via the `database` binary;
|
||||||
|
`db-ops` provides target-safe operational helpers such as secrets hydration.
|
||||||
|
|
||||||
## Provided Binaries
|
## Provided Binaries
|
||||||
|
|
||||||
| Binary | Description |
|
| Binary | Description |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `database` | Main script for managing migrations, deployments, and logs. |
|
| `database` | Main `db-dev` script for managing local development databases, migrations, deployments, and logs. |
|
||||||
|
| `db-ops` | Operational helper binary for target/runtime database actions. |
|
||||||
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
|
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
|
||||||
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
|
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
|
||||||
|
|
||||||
## Required Environment Variables
|
## Required Environment Variables
|
||||||
|
|
||||||
These variables must be set for `db-tool` to function.
|
These variables must be set for `db-dev` / `database` to function.
|
||||||
|
|
||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
@@ -39,7 +42,7 @@ These variables must be set for `db-tool` to function.
|
|||||||
|
|
||||||
## Postgres Package Override
|
## Postgres Package Override
|
||||||
|
|
||||||
By default, `db-tool`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
|
By default, `db-dev`/`db-ops`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
let
|
let
|
||||||
@@ -48,7 +51,8 @@ let
|
|||||||
]);
|
]);
|
||||||
in {
|
in {
|
||||||
packages = [
|
packages = [
|
||||||
(pkgs.hectic."db-tool".override { postgresql = myPg; })
|
(pkgs.hectic."db-dev".override { postgresql = myPg; })
|
||||||
|
(pkgs.hectic."db-ops".override { postgresql = myPg; })
|
||||||
(pkgs.hectic."postgres-init".override { postgresql = myPg; })
|
(pkgs.hectic."postgres-init".override { postgresql = myPg; })
|
||||||
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
|
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
|
||||||
];
|
];
|
||||||
@@ -102,14 +106,14 @@ used by `database restore`.
|
|||||||
|
|
||||||
## shellHook Example
|
## shellHook Example
|
||||||
|
|
||||||
To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
|
To use `db-dev` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
{
|
{
|
||||||
# ...
|
# ...
|
||||||
devShells.default = pkgs.mkShell {
|
devShells.default = pkgs.mkShell {
|
||||||
packages = [
|
packages = [
|
||||||
pkgs.hectic.db-tool
|
pkgs.hectic.db-dev
|
||||||
pkgs.hectic.postgres-init
|
pkgs.hectic.postgres-init
|
||||||
pkgs.hectic.postgres-cleanup
|
pkgs.hectic.postgres-cleanup
|
||||||
];
|
];
|
||||||
@@ -134,7 +138,7 @@ To use `db-tool` in a Nix development shell, add the following to your `flake.ni
|
|||||||
|
|
||||||
## hectic Bundle
|
## hectic Bundle
|
||||||
|
|
||||||
`db-tool` and `migrator` apply a single bundle of SQL files that bootstrap the
|
`db-dev`, `db-ops`, and `migrator` apply a single bundle of SQL files that bootstrap the
|
||||||
`hectic` schema. The bundle lives in
|
`hectic` schema. The bundle lives in
|
||||||
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
|
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
|
||||||
contract, file layout, and the `self.lib.hectic.*` Nix API.
|
contract, file layout, and the `self.lib.hectic.*` Nix API.
|
||||||
@@ -190,6 +194,7 @@ ALTER DATABASE mydb SET hectic.inheritance_extra_excluded_schemas = 'legacy,etl'
|
|||||||
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
|
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
|
||||||
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
|
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
|
||||||
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
|
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
|
||||||
|
| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. |
|
||||||
|
|
||||||
The bundle is idempotent — repeated application is safe.
|
The bundle is idempotent — repeated application is safe.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,222 @@
|
|||||||
|
# shellcheck shell=dash
|
||||||
|
|
||||||
|
: "${SCRIPT_NAME:=$(basename "$0")}"
|
||||||
|
|
||||||
|
help() {
|
||||||
|
# shellcheck disable=SC2059
|
||||||
|
printf "$(cat <<EOF
|
||||||
|
${BGREEN}Usage:${NC} $SCRIPT_NAME [OPTIONS] <SUBCOMMAND> [OPTIONS]
|
||||||
|
|
||||||
|
PostgreSQL operations utility.
|
||||||
|
|
||||||
|
${BGREEN}Global Options:${NC}
|
||||||
|
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
|
||||||
|
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
|
||||||
|
|
||||||
|
${BGREEN}Subcommands:${NC}
|
||||||
|
${BCYAN}secrets load${NC} [OPTIONS] Apply hectic bundle and load secrets
|
||||||
|
|
||||||
|
${BGREEN}Environment:${NC}
|
||||||
|
${BBLACK}PGURL${NC} PostgreSQL connection string fallback
|
||||||
|
${BBLACK}DB_URL${NC} PostgreSQL connection string fallback
|
||||||
|
${BBLACK}HECTIC_DOTENV_CONTENT${NC} Raw dotenv content to load
|
||||||
|
${BBLACK}HECTIC_DOTENV_FILE${NC} Dotenv file to read when readable
|
||||||
|
${BBLACK}LOCAL_DIR${NC} Used with ENVIRONMENT for .env fallback
|
||||||
|
${BBLACK}ENVIRONMENT${NC} Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
)"
|
||||||
|
}
|
||||||
|
|
||||||
|
help_secrets_load() {
|
||||||
|
# shellcheck disable=SC2059
|
||||||
|
printf "$(cat <<EOF
|
||||||
|
${BGREEN}Usage:${NC} $SCRIPT_NAME ${BCYAN}secrets load${NC} [OPTIONS]
|
||||||
|
|
||||||
|
Apply the hectic SQL bundle and load dotenv-backed secrets into
|
||||||
|
${BBLACK}hectic.secret${NC}.
|
||||||
|
|
||||||
|
${BGREEN}Options:${NC}
|
||||||
|
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
|
||||||
|
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
|
||||||
|
${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} <path> Dotenv file path
|
||||||
|
|
||||||
|
${BGREEN}Resolution order:${NC}
|
||||||
|
1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC}
|
||||||
|
2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC}
|
||||||
|
3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
|
||||||
|
|
||||||
|
Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv
|
||||||
|
source can be resolved.
|
||||||
|
|
||||||
|
EOF
|
||||||
|
)"
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_pgurl() {
|
||||||
|
if [ -n "${PGURL:-}" ]; then
|
||||||
|
printf '%s' "$PGURL"
|
||||||
|
elif [ -n "${DB_URL:-}" ]; then
|
||||||
|
printf '%s' "$DB_URL"
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_dotenv_content() {
|
||||||
|
dotenv_file="${1:-}"
|
||||||
|
|
||||||
|
if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then
|
||||||
|
printf '%s' "$HECTIC_DOTENV_CONTENT"
|
||||||
|
elif [ -n "$dotenv_file" ]; then
|
||||||
|
if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
cat "$dotenv_file"
|
||||||
|
elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
|
||||||
|
cat "${LOCAL_DIR}/.env.${ENVIRONMENT}"
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
subcommand_secrets_load() {
|
||||||
|
change_namespace 'db ops secrets load'
|
||||||
|
|
||||||
|
dotenv_file="${HECTIC_DOTENV_FILE:-}"
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
-h|--help)
|
||||||
|
help_secrets_load
|
||||||
|
restore_namespace
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-u|--url)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "missing value for $1"
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
PGURL=$2
|
||||||
|
DB_URL=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-f|--dotenv-file)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "missing value for $1"
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
dotenv_file=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--*|-*)
|
||||||
|
log error "secrets load argument $1 does not exist"
|
||||||
|
restore_namespace
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log error "secrets load subcommand $1 does not exist"
|
||||||
|
restore_namespace
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! pgurl="$(resolve_pgurl)"; then
|
||||||
|
log error "PGURL or DB_URL is required"
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then
|
||||||
|
:
|
||||||
|
else
|
||||||
|
dotenv_content_exit_code=$?
|
||||||
|
if [ "$dotenv_content_exit_code" -eq 2 ]; then
|
||||||
|
log error "dotenv file is not readable: $dotenv_file"
|
||||||
|
else
|
||||||
|
log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)"
|
||||||
|
fi
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
log notice "apply hectic bundle and load secrets"
|
||||||
|
apply_hectic_bundle "$pgurl" "$dotenv_content"
|
||||||
|
restore_namespace
|
||||||
|
}
|
||||||
|
|
||||||
|
subcommand_secrets() {
|
||||||
|
change_namespace 'db ops secrets'
|
||||||
|
|
||||||
|
if [ $# -eq 0 ]; then
|
||||||
|
help_secrets_load
|
||||||
|
restore_namespace
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
subcommand=$1
|
||||||
|
shift
|
||||||
|
|
||||||
|
case $subcommand in
|
||||||
|
load)
|
||||||
|
subcommand_secrets_load "$@"
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
help_secrets_load
|
||||||
|
restore_namespace
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log error "secrets subcommand $subcommand does not exist"
|
||||||
|
restore_namespace
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
-h|--help)
|
||||||
|
help
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-u|--url)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "missing value for $1"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
PGURL=$2
|
||||||
|
DB_URL=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--*|-*)
|
||||||
|
log error "argument $1 does not exist"
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
subcommand="${1:-}"
|
||||||
|
|
||||||
|
if [ -z "$subcommand" ]; then
|
||||||
|
help
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
shift
|
||||||
|
|
||||||
|
case $subcommand in
|
||||||
|
secrets)
|
||||||
|
subcommand_secrets "$@"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log error "subcommand $subcommand does not exist"
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
esac
|
||||||
@@ -8,7 +8,7 @@ let
|
|||||||
|
|
||||||
applyBundle = self.lib.hectic.applyBundleScript;
|
applyBundle = self.lib.hectic.applyBundleScript;
|
||||||
|
|
||||||
mkDatabase =
|
mkDbDev =
|
||||||
{ postgresql ? postgresql_17 }:
|
{ postgresql ? postgresql_17 }:
|
||||||
hectic.writeShellApplication {
|
hectic.writeShellApplication {
|
||||||
inherit shell;
|
inherit shell;
|
||||||
@@ -27,7 +27,7 @@ let
|
|||||||
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
|
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
|
||||||
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
|
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
|
||||||
${applyBundle}
|
${applyBundle}
|
||||||
${builtins.readFile ./database.sh}
|
${builtins.readFile ./db-dev.sh}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
@@ -36,6 +36,31 @@ let
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mkDbOps =
|
||||||
|
{ postgresql ? postgresql_17 }:
|
||||||
|
hectic.writeShellApplication {
|
||||||
|
inherit shell;
|
||||||
|
bashOptions = [
|
||||||
|
"errexit"
|
||||||
|
"nounset"
|
||||||
|
];
|
||||||
|
excludeShellChecks = [ "SC2209" ];
|
||||||
|
name = "db-ops";
|
||||||
|
runtimeInputs = [ postgresql coreutils ];
|
||||||
|
|
||||||
|
text = ''
|
||||||
|
${builtins.readFile hectic.helpers.posix-shell.log}
|
||||||
|
${builtins.readFile hectic.helpers.posix-shell.change_namespace}
|
||||||
|
${applyBundle}
|
||||||
|
${builtins.readFile ./db-ops.sh}
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "PostgreSQL operations utility";
|
||||||
|
mainProgram = "db-ops";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
mkPostgresInit =
|
mkPostgresInit =
|
||||||
{ postgresql ? postgresql_17 }:
|
{ postgresql ? postgresql_17 }:
|
||||||
hectic.writeShellApplication {
|
hectic.writeShellApplication {
|
||||||
@@ -73,7 +98,8 @@ let
|
|||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
"db-tool" = lib.makeOverridable mkDatabase { };
|
"db-dev" = lib.makeOverridable mkDbDev { };
|
||||||
|
"db-ops" = lib.makeOverridable mkDbOps { };
|
||||||
"postgres-init" = lib.makeOverridable mkPostgresInit { };
|
"postgres-init" = lib.makeOverridable mkPostgresInit { };
|
||||||
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
|
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
|
||||||
"hectic-inheritance" = hecticInheritance;
|
"hectic-inheritance" = hecticInheritance;
|
||||||
|
|||||||
@@ -1,9 +1,46 @@
|
|||||||
#!/bin/dash
|
#!/bin/dash
|
||||||
|
|
||||||
|
# Stop a local PostgreSQL cluster started by postgres-init.
|
||||||
|
#
|
||||||
|
# Public contract:
|
||||||
|
# - Accepts PG_WORKING_DIR directly, or derives it from LOCAL_DIR.
|
||||||
|
# - If no cluster root can be resolved, exits successfully without doing
|
||||||
|
# anything. This keeps cleanup safe in traps and partial-init flows.
|
||||||
|
# - If no postmaster.pid exists, treats the cluster as already stopped.
|
||||||
|
# - NO_TTY=1 redirects pg_ctl output into a temp log file instead of writing to
|
||||||
|
# the current terminal.
|
||||||
|
|
||||||
|
postgres_cleanup_help() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: postgres-cleanup
|
||||||
|
|
||||||
|
Stop a local PostgreSQL cluster if it is running.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
PG_WORKING_DIR Cluster root directory
|
||||||
|
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
|
||||||
|
NO_TTY Redirect pg_ctl output to a temp file
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- Returns success if the cluster directory cannot be resolved.
|
||||||
|
- Returns success if postmaster.pid is already absent.
|
||||||
|
- Ignores pg_ctl stop errors so cleanup remains trap-safe.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
postgres_cleanup_main() {
|
postgres_cleanup_main() {
|
||||||
|
case "${1:-}" in
|
||||||
|
-h|--help)
|
||||||
|
postgres_cleanup_help
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
|
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
|
||||||
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
|
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
|
||||||
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
|
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
|
||||||
|
# Cleanup is intentionally forgiving so it can be used in traps after
|
||||||
|
# partial startup failures without masking the real error.
|
||||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||||
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
|
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
|
||||||
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
|
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
|
||||||
|
|||||||
@@ -1,6 +1,58 @@
|
|||||||
#!/bin/dash
|
#!/bin/dash
|
||||||
|
|
||||||
|
# Initialize or reuse a local PostgreSQL cluster for development workflows.
|
||||||
|
#
|
||||||
|
# Public contract:
|
||||||
|
# - Requires either PG_WORKING_DIR or LOCAL_DIR.
|
||||||
|
# - Produces a ready-to-use database and exports:
|
||||||
|
# POSTGRESQL_HOST, POSTGRESQL_PORT, POSTGRESQL_USER, POSTGRESQL_DATABASE,
|
||||||
|
# PGURL, and also the variable named by PG_URL_VAR.
|
||||||
|
# - Reuses an existing cluster only when PG_REUSE is set and PG_VERSION exists.
|
||||||
|
# - If a reused cluster cannot start, it is reinitialized automatically.
|
||||||
|
#
|
||||||
|
# Important knobs:
|
||||||
|
# - PG_WORKING_DIR: cluster root (defaults to $LOCAL_DIR/focus/postgresql)
|
||||||
|
# - PG_DATABASE: database to create/ensure (default: testdb)
|
||||||
|
# - PG_PORT: socket port (default: 5432)
|
||||||
|
# - PG_URL_VAR: alternate URL variable to export in addition to PGURL
|
||||||
|
# - PG_CONF_FILE: base postgresql.conf to copy on fresh init
|
||||||
|
# - PG_SHARED_PRELOAD_LIBRARIES: preload list; empty string disables pg_cron
|
||||||
|
# - PG_DISABLE_LOGGING=1: disable logging_collector in generated config
|
||||||
|
# - NO_TTY=1: redirect pg_ctl stop/start output into temp log files
|
||||||
|
|
||||||
|
postgres_init_help() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: postgres-init
|
||||||
|
|
||||||
|
Initialize or reuse a local PostgreSQL cluster.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
PG_WORKING_DIR Cluster root directory
|
||||||
|
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
|
||||||
|
PG_DATABASE Database name to create/ensure (default: testdb)
|
||||||
|
PG_PORT PostgreSQL port / unix socket port (default: 5432)
|
||||||
|
PG_URL_VAR Extra URL variable to export alongside PGURL
|
||||||
|
PG_CONF_FILE Base postgresql.conf copied on fresh init only
|
||||||
|
PG_SHARED_PRELOAD_LIBRARIES Preload list; empty disables pg_cron preload
|
||||||
|
PG_DISABLE_LOGGING Set to 1 to disable logging_collector
|
||||||
|
PG_REUSE Reuse existing cluster if PG_VERSION exists
|
||||||
|
NO_TTY Redirect pg_ctl output to temp files
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- Rewrites runtime socket/port/cron settings on every launch.
|
||||||
|
- Creates the target database if missing.
|
||||||
|
- If a reused cluster exists but cannot start, reinitializes it automatically.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
postgres_init_main() {
|
postgres_init_main() {
|
||||||
|
case "${1:-}" in
|
||||||
|
-h|--help)
|
||||||
|
postgres_init_help
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
|
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
|
||||||
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
|
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
|
||||||
return 1
|
return 1
|
||||||
@@ -25,20 +77,41 @@ postgres_init_main() {
|
|||||||
fi
|
fi
|
||||||
mkdir -p "$sockdir" || return 1
|
mkdir -p "$sockdir" || return 1
|
||||||
|
|
||||||
|
# Reuse is optimistic: if a cluster exists on disk, try to start it first.
|
||||||
|
# We only destroy state after a real startup failure proves the cluster is
|
||||||
|
# broken, which keeps long-lived local DBs intact when possible.
|
||||||
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
|
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
|
||||||
|
|
||||||
if [ "$PG_REUSE" -eq 0 ]; then
|
if [ "$PG_REUSE" -eq 0 ]; then
|
||||||
rm -rf "$data" "$sockdir" || return 1
|
rm -rf "$data" "$sockdir" || return 1
|
||||||
mkdir -p "$sockdir" || return 1
|
mkdir -p "$sockdir" || return 1
|
||||||
initdb -D "$data" --no-locale -E UTF8 || return 1
|
initdb -D "$data" --no-locale -E UTF8 || return 1
|
||||||
if [ -n "${PG_CONF_FILE:-}" ]; then
|
if [ -n "${PG_CONF_FILE:-}" ]; then
|
||||||
|
# PG_CONF_FILE replaces the base postgresql.conf on fresh init only. We
|
||||||
|
# still append runtime values later so the helper can relocate sockets,
|
||||||
|
# ports, and cron settings regardless of the custom file contents.
|
||||||
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
||||||
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
||||||
else
|
else
|
||||||
{ printf '%s\n' "listen_addresses = ''"; [ "$PG_DISABLE_LOGGING" -eq 0 ] && { printf '%s\n' 'logging_collector = on'; printf '%s\n' "log_directory = 'log'"; }; [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ] && { printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"; printf '%s\n' "cron.database_name = '$db'"; printf '%s\n' "cron.host = '$sockdir'"; }; :; } >> "$data/postgresql.conf" || return 1
|
{
|
||||||
|
printf '%s\n' "listen_addresses = ''"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' 'logging_collector = on'
|
||||||
|
printf '%s\n' "log_directory = 'log'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.conf" || return 1
|
||||||
fi
|
fi
|
||||||
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Rewrite the runtime knobs on every launch. Reused clusters may have been
|
||||||
|
# started from another workspace/session, so we must override stale socket,
|
||||||
|
# port, and pg_cron wiring before attempting startup.
|
||||||
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
||||||
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
||||||
|
|
||||||
@@ -68,12 +141,85 @@ postgres_init_main() {
|
|||||||
printf '%s\n' "cron.host = '$sockdir'"
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
fi
|
fi
|
||||||
} >> "$data/postgresql.auto.conf" || return 1
|
} >> "$data/postgresql.auto.conf" || return 1
|
||||||
|
|
||||||
|
_pg_start_exit=0
|
||||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||||
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
||||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || return 2
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
|
||||||
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
||||||
else
|
else
|
||||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || return 2
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$_pg_start_exit" -ne 0 ]; then
|
||||||
|
if [ "$PG_REUSE" -eq 1 ]; then
|
||||||
|
# Self-heal path: the on-disk cluster exists but cannot complete startup
|
||||||
|
# (for example bad WAL / invalid checkpoint). At this point preserving
|
||||||
|
# the broken state is less useful than reinitializing automatically.
|
||||||
|
printf '%s\n' "postgres-init: reused cluster failed to start; reinitializing $wd" >&2
|
||||||
|
PG_REUSE=0
|
||||||
|
rm -rf "$data" "$sockdir" || return 1
|
||||||
|
mkdir -p "$sockdir" || return 1
|
||||||
|
initdb -D "$data" --no-locale -E UTF8 || return 1
|
||||||
|
if [ -n "${PG_CONF_FILE:-}" ]; then
|
||||||
|
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
||||||
|
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
||||||
|
else
|
||||||
|
{
|
||||||
|
printf '%s\n' "listen_addresses = ''"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' 'logging_collector = on'
|
||||||
|
printf '%s\n' "log_directory = 'log'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.conf" || return 1
|
||||||
|
fi
|
||||||
|
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
||||||
|
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
||||||
|
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
||||||
|
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
|
||||||
|
{
|
||||||
|
printf '%s\n' "port = '$PG_PORT'"
|
||||||
|
printf '%s\n' "unix_socket_directories = '$sockdir'"
|
||||||
|
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
|
||||||
|
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' "logging_collector = 'on'"
|
||||||
|
else
|
||||||
|
printf '%s\n' "logging_collector = 'off'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.auto.conf" || return 1
|
||||||
|
_pg_start_exit=0
|
||||||
|
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||||
|
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
||||||
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
|
||||||
|
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
||||||
|
else
|
||||||
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
|
||||||
|
fi
|
||||||
|
[ "$_pg_start_exit" -eq 0 ] || return 2
|
||||||
|
printf '%s\n' "postgres-init: reinitialized broken cluster at $wd" >&2
|
||||||
|
else
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
user="$(id -un)" || return 1
|
user="$(id -un)" || return 1
|
||||||
@@ -87,9 +233,13 @@ postgres_init_main() {
|
|||||||
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
|
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
|
||||||
|
|
||||||
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
|
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
|
||||||
_pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
|
# Export both names for compatibility: some callers consume plain PGURL,
|
||||||
|
# while multi-cluster shells also need a project-specific variable like
|
||||||
|
# WIPE_PGURL or BMSEARCH_PGURL in the same environment.
|
||||||
|
PGURL="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
|
||||||
|
export PGURL
|
||||||
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
|
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
|
||||||
export "${PG_URL_VAR}=${_pg_url}" || return 1
|
export "${PG_URL_VAR}=${PGURL}" || return 1
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+6
-1
@@ -132,6 +132,7 @@ in {
|
|||||||
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
|
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
|
||||||
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
|
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
|
||||||
migration-name = pkgs.callPackage ./migration-name.nix {};
|
migration-name = pkgs.callPackage ./migration-name.nix {};
|
||||||
|
plgo = pkgs.callPackage ./plgo {};
|
||||||
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
|
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
|
||||||
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
|
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
|
||||||
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
|
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
|
||||||
@@ -142,15 +143,18 @@ in {
|
|||||||
watch = pkgs.callPackage ./c/watch/default.nix {};
|
watch = pkgs.callPackage ./c/watch/default.nix {};
|
||||||
support-bot = pkgs.callPackage ./support-bot {};
|
support-bot = pkgs.callPackage ./support-bot {};
|
||||||
gitea-heatmap = pkgs.callPackage ./gitea {};
|
gitea-heatmap = pkgs.callPackage ./gitea {};
|
||||||
|
gitea-runner-nix-image = pkgs.callPackage ./gitea-runner-nix-image {};
|
||||||
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
|
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
|
||||||
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
|
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
|
||||||
deploy = pkgs.callPackage ./deploy { inherit inputs; };
|
deploy = pkgs.callPackage ./deploy { inherit inputs; };
|
||||||
element-web = pkgs.callPackage ./element-web {};
|
element-web = pkgs.callPackage ./element-web {};
|
||||||
shellplot = pkgs.callPackage ./shellplot {};
|
shellplot = pkgs.callPackage ./shellplot {};
|
||||||
|
which-country-rs = pkgs.callPackage ./which-country-rs {};
|
||||||
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
|
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
|
||||||
migrator = pkgs.callPackage ./migrator { inherit self; };
|
migrator = pkgs.callPackage ./migrator { inherit self; };
|
||||||
"parse-uri" = pkgs.callPackage ./parse-uri {};
|
"parse-uri" = pkgs.callPackage ./parse-uri {};
|
||||||
"db-tool" = dbToolPkgs."db-tool";
|
"db-dev" = dbToolPkgs."db-dev";
|
||||||
|
"db-ops" = dbToolPkgs."db-ops";
|
||||||
"postgres-init" = dbToolPkgs."postgres-init";
|
"postgres-init" = dbToolPkgs."postgres-init";
|
||||||
"postgres-cleanup" = dbToolPkgs."postgres-cleanup";
|
"postgres-cleanup" = dbToolPkgs."postgres-cleanup";
|
||||||
"hectic-inheritance" = dbToolPkgs."hectic-inheritance";
|
"hectic-inheritance" = dbToolPkgs."hectic-inheritance";
|
||||||
@@ -172,5 +176,6 @@ in {
|
|||||||
pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
|
pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
|
||||||
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
|
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
|
||||||
pg-15-ext-plsh = buildPlShExt pkgs "15";
|
pg-15-ext-plsh = buildPlShExt pkgs "15";
|
||||||
|
stable-video-diffusion-api = pkgs.callPackage ./stable-video-diffusion-api {};
|
||||||
media-browser = pkgs.callPackage ./media-browser {};
|
media-browser = pkgs.callPackage ./media-browser {};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ let
|
|||||||
hash = "sha256-pbzuPgKJ0DmrDSTO7ZTDArX+Xr9k/ndAGZvQg2kMTMQ=";
|
hash = "sha256-pbzuPgKJ0DmrDSTO7ZTDArX+Xr9k/ndAGZvQg2kMTMQ=";
|
||||||
};
|
};
|
||||||
|
|
||||||
inherit patches;
|
#inherit patches; #WIP
|
||||||
|
|
||||||
pnpmDeps = fetchPnpmDeps {
|
pnpmDeps = fetchPnpmDeps {
|
||||||
pname = "element";
|
pname = "element";
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
{
|
||||||
|
bash,
|
||||||
|
cacert,
|
||||||
|
coreutils,
|
||||||
|
dockerTools,
|
||||||
|
git,
|
||||||
|
lib,
|
||||||
|
nix,
|
||||||
|
symlinkJoin,
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
name = "gitea-runner-nix-image";
|
||||||
|
tag = "2026-06-07";
|
||||||
|
root = symlinkJoin {
|
||||||
|
name = "${name}-root";
|
||||||
|
paths = [
|
||||||
|
bash
|
||||||
|
cacert
|
||||||
|
coreutils
|
||||||
|
git
|
||||||
|
nix
|
||||||
|
];
|
||||||
|
};
|
||||||
|
in
|
||||||
|
dockerTools.buildLayeredImageWithNixDb {
|
||||||
|
inherit name tag;
|
||||||
|
|
||||||
|
contents = [ root ];
|
||||||
|
|
||||||
|
fakeRootCommands = ''
|
||||||
|
mkdir -p ./etc
|
||||||
|
cat > ./etc/passwd <<'EOF'
|
||||||
|
root:x:0:0:root:/root:/bin/bash
|
||||||
|
nobody:x:65534:65534:nobody:/var/empty:/bin/false
|
||||||
|
EOF
|
||||||
|
cat > ./etc/group <<'EOF'
|
||||||
|
root:x:0:
|
||||||
|
nixbld:x:30000:
|
||||||
|
nobody:x:65534:
|
||||||
|
EOF
|
||||||
|
for n in $(seq 1 10); do
|
||||||
|
echo "nixbld$n:x:$((30000 + n)):30000:Nix build user $n:/var/empty:/bin/false" >> ./etc/passwd
|
||||||
|
sed -i "s/^nixbld:x:30000:.*/&,nixbld$n/" ./etc/group
|
||||||
|
done
|
||||||
|
|
||||||
|
mkdir -p ./nix/var/nix/{gcroots,profiles,temproots,userpool,db}
|
||||||
|
mkdir -p ./nix/var/log/nix/drvs
|
||||||
|
chmod 1777 ./nix/var/nix/gcroots ./nix/var/nix/profiles
|
||||||
|
'';
|
||||||
|
|
||||||
|
extraCommands = ''
|
||||||
|
mkdir -p etc/nix root tmp
|
||||||
|
chmod 1777 tmp
|
||||||
|
|
||||||
|
cat > etc/nix/nix.conf <<'EOF'
|
||||||
|
accept-flake-config = true
|
||||||
|
experimental-features = nix-command flakes
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||||
|
trusted-users = root
|
||||||
|
sandbox = false
|
||||||
|
EOF
|
||||||
|
'';
|
||||||
|
|
||||||
|
config = {
|
||||||
|
Cmd = [ "${bash}/bin/bash" ];
|
||||||
|
Env = [
|
||||||
|
"HOME=/root"
|
||||||
|
"PATH=${lib.makeBinPath [ bash coreutils git nix ]}"
|
||||||
|
"SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||||
|
"NIX_SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||||
|
"USER=root"
|
||||||
|
];
|
||||||
|
Labels = {
|
||||||
|
"org.opencontainers.image.description" = "Nix-capable Gitea Actions job image";
|
||||||
|
"org.opencontainers.image.ref.name" = "${name}:${tag}";
|
||||||
|
"org.opencontainers.image.source" = "https://gitea.hectic-lab.com/hectic-lab/util.nix";
|
||||||
|
};
|
||||||
|
WorkingDir = "/workspace";
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
|
|||||||
Commands: []*cli.Command{
|
Commands: []*cli.Command{
|
||||||
newUserCommand(),
|
newUserCommand(),
|
||||||
newRepoSyncReleasesCommand(),
|
newRepoSyncReleasesCommand(),
|
||||||
|
newHeatmapCommand(),
|
||||||
newRegenerateCommand(),
|
newRegenerateCommand(),
|
||||||
newAuthCommand(),
|
newAuthCommand(),
|
||||||
newSendMailCommand(),
|
newSendMailCommand(),
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
repo_service "code.gitea.io/gitea/services/repository"
|
||||||
|
|
||||||
|
"github.com/urfave/cli/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newHeatmapCommand() *cli.Command {
|
||||||
|
return &cli.Command{
|
||||||
|
Name: "heatmap",
|
||||||
|
Usage: "Manage heatmap indexes",
|
||||||
|
Commands: []*cli.Command{
|
||||||
|
newHeatmapReindexCommand(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHeatmapReindexCommand() *cli.Command {
|
||||||
|
return &cli.Command{
|
||||||
|
Name: "reindex",
|
||||||
|
Usage: "Reindex heatmap contributions from repository default branches",
|
||||||
|
Action: runHeatmapReindex,
|
||||||
|
Flags: []cli.Flag{
|
||||||
|
&cli.BoolFlag{
|
||||||
|
Name: "all",
|
||||||
|
Usage: "Reindex all repositories",
|
||||||
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
Name: "repo",
|
||||||
|
Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
|
||||||
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
Name: "owner",
|
||||||
|
Usage: "Owner name for --repo",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
|
||||||
|
if err := initDB(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := git.InitSimple(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
reindexAll := c.Bool("all")
|
||||||
|
repoName := c.String("repo")
|
||||||
|
ownerName := c.String("owner")
|
||||||
|
if reindexAll {
|
||||||
|
if repoName != "" || ownerName != "" {
|
||||||
|
return fmt.Errorf("--all cannot be combined with --repo or --owner")
|
||||||
|
}
|
||||||
|
return reindexAllHeatmapRepositories(ctx)
|
||||||
|
}
|
||||||
|
if repoName == "" {
|
||||||
|
return fmt.Errorf("either --all or --repo must be provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return reindexHeatmapRepository(ctx, repo)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
|
||||||
|
if ownerName != "" {
|
||||||
|
if strings.Contains(repoName, "/") {
|
||||||
|
return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
|
||||||
|
}
|
||||||
|
return ownerName, repoName, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ownerName, repoName, ok := strings.Cut(repoName, "/")
|
||||||
|
if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
|
||||||
|
return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
|
||||||
|
}
|
||||||
|
return ownerName, repoName, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reindexAllHeatmapRepositories(ctx context.Context) error {
|
||||||
|
for page := 1; ; page++ {
|
||||||
|
repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
|
||||||
|
ListOptions: db.ListOptions{
|
||||||
|
PageSize: repo_model.RepositoryListDefaultPageSize,
|
||||||
|
Page: page,
|
||||||
|
},
|
||||||
|
Private: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("SearchRepositoryByName: %w", err)
|
||||||
|
}
|
||||||
|
if len(repos) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
log.Trace("Processing next %d repos of %d", len(repos), count)
|
||||||
|
for _, repo := range repos {
|
||||||
|
if err := reindexHeatmapRepository(ctx, repo); err != nil {
|
||||||
|
log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
|
||||||
|
log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
|
||||||
|
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
repo_service "code.gitea.io/gitea/services/repository"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeatmapAdminReindex(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
|
||||||
|
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||||
|
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||||
|
contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
|
||||||
|
|
||||||
|
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
|
||||||
|
newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
|
||||||
|
{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
})
|
||||||
|
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||||
|
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
|
||||||
|
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
|
||||||
|
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeatmapAdminRepoSelector(t *testing.T) {
|
||||||
|
owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "user", owner)
|
||||||
|
assert.Equal(t, "repo", repo)
|
||||||
|
|
||||||
|
owner, repo, err = parseHeatmapRepoSelector("user", "repo")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "user", owner)
|
||||||
|
assert.Equal(t, "repo", repo)
|
||||||
|
|
||||||
|
_, _, err = parseHeatmapRepoSelector("", "repo")
|
||||||
|
assert.ErrorContains(t, err, "owner/name")
|
||||||
|
_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
|
||||||
|
assert.ErrorContains(t, err, "when --owner is set")
|
||||||
|
}
|
||||||
|
|
||||||
|
type heatmapAdminTestCommit struct {
|
||||||
|
Branch string
|
||||||
|
Mark string
|
||||||
|
Parent string
|
||||||
|
AuthorName string
|
||||||
|
AuthorEmail string
|
||||||
|
CommitterName string
|
||||||
|
CommitterEmail string
|
||||||
|
AuthorDate string
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
|
||||||
|
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
|
repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
marks := runHeatmapAdminFastImport(t, repo, commits)
|
||||||
|
repo.IsEmpty = false
|
||||||
|
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
|
||||||
|
return repo, marks
|
||||||
|
}
|
||||||
|
|
||||||
|
func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var stream strings.Builder
|
||||||
|
branchTips := make(map[string]string)
|
||||||
|
for i, commit := range commits {
|
||||||
|
mark := fmt.Sprintf(":%d", i+1)
|
||||||
|
branchRef := "refs/heads/" + commit.Branch
|
||||||
|
stream.WriteString("commit " + branchRef + "\n")
|
||||||
|
stream.WriteString("mark " + mark + "\n")
|
||||||
|
stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
|
||||||
|
stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
|
||||||
|
message := "heatmap admin " + commit.Mark
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
|
||||||
|
if parentMark := branchTips[commit.Branch]; parentMark != "" {
|
||||||
|
stream.WriteString("from " + parentMark + "\n")
|
||||||
|
} else if commit.Parent != "" {
|
||||||
|
stream.WriteString("from " + commit.Parent + "\n")
|
||||||
|
}
|
||||||
|
content := commit.Mark + "\n"
|
||||||
|
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
|
||||||
|
branchTips[commit.Branch] = mark
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
WithStdinCopy(strings.NewReader(stream.String())).
|
||||||
|
Run(t.Context()))
|
||||||
|
|
||||||
|
commitSHAs := make(map[string]string, len(commits))
|
||||||
|
for _, commit := range commits {
|
||||||
|
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
|
||||||
|
AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
|
||||||
|
AddDynamicArguments("refs/heads/" + commit.Branch).
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
RunStdString(t.Context())
|
||||||
|
require.NoError(t, err)
|
||||||
|
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
|
||||||
|
}
|
||||||
|
return commitSHAs
|
||||||
|
}
|
||||||
|
|
||||||
|
func heatmapAdminSignatureLine(kind, name, email, date string) string {
|
||||||
|
parsed, err := time.Parse(time.RFC3339, date)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
|
||||||
|
t.Helper()
|
||||||
|
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return contributions
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package activities
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"xorm.io/xorm/schemas"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HeatmapContribution stores commit contributions counted for profile heatmaps.
|
||||||
|
type HeatmapContribution struct {
|
||||||
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
UserID int64 `xorm:"NOT NULL"`
|
||||||
|
RepoID int64 `xorm:"NOT NULL"`
|
||||||
|
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
|
||||||
|
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
|
||||||
|
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
|
||||||
|
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||||
|
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing.
|
||||||
|
type HeatmapContributionIdentity struct {
|
||||||
|
RepoID int64
|
||||||
|
CommitSHA string
|
||||||
|
UserID int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
db.RegisterModel(new(HeatmapContribution))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TableIndices implements xorm's TableIndices interface.
|
||||||
|
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
|
||||||
|
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
|
||||||
|
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
|
||||||
|
|
||||||
|
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
|
||||||
|
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
|
||||||
|
|
||||||
|
return []*schemas.Index{uniqueContribution, userAuthorRepo}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertHeatmapContribution creates or updates a commit contribution without duplicating counted rows.
|
||||||
|
func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContribution) error {
|
||||||
|
return db.WithTx(ctx, func(ctx context.Context) error {
|
||||||
|
e := db.GetEngine(ctx)
|
||||||
|
|
||||||
|
rows, err := e.Where("repo_id=? AND commit_sha=? AND user_id=?", contribution.RepoID, contribution.CommitSHA, contribution.UserID).
|
||||||
|
Cols("author_email", "author_unix").
|
||||||
|
Update(contribution)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if rows > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
has, err := e.Exist(&HeatmapContribution{RepoID: contribution.RepoID, CommitSHA: contribution.CommitSHA, UserID: contribution.UserID})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if has {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = e.Insert(contribution)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountHeatmapContributions returns counted heatmap contribution rows for a user.
|
||||||
|
func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) {
|
||||||
|
return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution))
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities.
|
||||||
|
func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error {
|
||||||
|
if len(identities) == 0 {
|
||||||
|
_, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
current := make(map[HeatmapContributionIdentity]struct{}, len(identities))
|
||||||
|
for _, identity := range identities {
|
||||||
|
current[identity] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
contributions, err := FindHeatmapContributionsByRepo(ctx, repoID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, contribution := range contributions {
|
||||||
|
identity := HeatmapContributionIdentity{
|
||||||
|
RepoID: contribution.RepoID,
|
||||||
|
CommitSHA: contribution.CommitSHA,
|
||||||
|
UserID: contribution.UserID,
|
||||||
|
}
|
||||||
|
if _, ok := current[identity]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository.
|
||||||
|
func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) {
|
||||||
|
contributions := make([]*HeatmapContribution, 0)
|
||||||
|
return contributions, db.GetEngine(ctx).
|
||||||
|
Where("repo_id=?", repoID).
|
||||||
|
OrderBy("author_unix ASC, commit_sha ASC, user_id ASC").
|
||||||
|
Find(&contributions)
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package activities_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeatmapContributionModel(t *testing.T) {
|
||||||
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
assert.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
|
||||||
|
const (
|
||||||
|
userID = int64(2)
|
||||||
|
repoID = int64(1)
|
||||||
|
commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
authorUnix = timeutil.TimeStamp(1579089600)
|
||||||
|
)
|
||||||
|
|
||||||
|
contribution := &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: "user2@example.com",
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), contribution))
|
||||||
|
|
||||||
|
count, err := activities_model.CountHeatmapContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.EqualValues(t, 1, count)
|
||||||
|
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: "changed@example.com",
|
||||||
|
AuthorUnix: authorUnix + 900,
|
||||||
|
}))
|
||||||
|
|
||||||
|
count, err = activities_model.CountHeatmapContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.EqualValues(t, 1, count)
|
||||||
|
|
||||||
|
stored := &activities_model.HeatmapContribution{UserID: userID, RepoID: repoID, CommitSHA: commitSHA}
|
||||||
|
has, err := db.GetEngine(t.Context()).Get(stored)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.True(t, has)
|
||||||
|
assert.Equal(t, "changed@example.com", stored.AuthorEmail)
|
||||||
|
assert.Equal(t, authorUnix+900, stored.AuthorUnix)
|
||||||
|
assert.NotZero(t, stored.CreatedUnix)
|
||||||
|
assert.NotZero(t, stored.UpdatedUnix)
|
||||||
|
|
||||||
|
err = db.Insert(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: "duplicate@example.com",
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
})
|
||||||
|
assert.Error(t, err)
|
||||||
|
|
||||||
|
includePrivate, err := user_model.GetIncludePrivateContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, includePrivate)
|
||||||
|
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), userID, true))
|
||||||
|
includePrivate, err = user_model.GetIncludePrivateContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, includePrivate)
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"code.gitea.io/gitea/models/organization"
|
"code.gitea.io/gitea/models/organization"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -38,35 +39,45 @@ func getUserHeatmapData(ctx context.Context, user *user_model.User, team *organi
|
|||||||
|
|
||||||
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
|
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
|
||||||
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
|
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
|
||||||
groupBy := "created_unix / 900 * 900"
|
groupBy := "author_unix / 900 * 900"
|
||||||
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
|
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
|
||||||
switch {
|
switch {
|
||||||
case setting.Database.Type.IsMySQL():
|
case setting.Database.Type.IsMySQL():
|
||||||
groupBy = "created_unix DIV 900 * 900"
|
groupBy = "author_unix DIV 900 * 900"
|
||||||
case setting.Database.Type.IsMSSQL():
|
case setting.Database.Type.IsMSSQL():
|
||||||
groupByName = groupBy
|
groupByName = groupBy
|
||||||
}
|
}
|
||||||
|
|
||||||
cond, err := ActivityQueryCondition(ctx, GetFeedsOptions{
|
includePrivate, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
|
||||||
RequestedUser: user,
|
|
||||||
RequestedTeam: team,
|
|
||||||
Actor: doer,
|
|
||||||
IncludePrivate: true, // don't filter by private, as we already filter by repo access
|
|
||||||
IncludeDeleted: true,
|
|
||||||
// * Heatmaps for individual users only include actions that the user themself did.
|
|
||||||
// * For organizations actions by all users that were made in owned
|
|
||||||
// repositories are counted.
|
|
||||||
OnlyPerformedBy: !user.IsOrganization(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return hdata, db.GetEngine(ctx).
|
sess := db.GetEngine(ctx).
|
||||||
Select(groupBy+" AS timestamp, count(user_id) as contributions").
|
Select(groupBy+" AS timestamp, count(heatmap_contribution.user_id) as contributions").
|
||||||
Table("action").
|
Table("heatmap_contribution").
|
||||||
Where(cond).
|
Join("INNER", "repository", "repository.id = heatmap_contribution.repo_id").
|
||||||
And("created_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
|
Join("INNER", "`user` AS repo_owner", "repo_owner.id = repository.owner_id").
|
||||||
|
Where("author_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
|
||||||
|
And("author_unix <= ?", timeutil.TimeStampNow())
|
||||||
|
|
||||||
|
if !includePrivate {
|
||||||
|
sess = sess.And("repository.is_private = ?", false).
|
||||||
|
And("repo_owner.visibility = ?", structs.VisibleTypePublic)
|
||||||
|
}
|
||||||
|
|
||||||
|
if user.IsOrganization() {
|
||||||
|
sess = sess.And("repository.owner_id = ?", user.ID)
|
||||||
|
if team != nil && !team.IncludesAllRepositories {
|
||||||
|
sess = sess.Join("INNER", "team_repo", "team_repo.repo_id = repository.id").
|
||||||
|
And("team_repo.org_id = ?", team.OrgID).
|
||||||
|
And("team_repo.team_id = ?", team.ID)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sess = sess.And("heatmap_contribution.user_id = ?", user.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hdata, sess.
|
||||||
GroupBy(groupByName).
|
GroupBy(groupByName).
|
||||||
OrderBy("timestamp").
|
OrderBy("timestamp").
|
||||||
Find(&hdata)
|
Find(&hdata)
|
||||||
|
|||||||
@@ -4,19 +4,43 @@
|
|||||||
package activities_test
|
package activities_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha1"
|
||||||
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
activities_model "code.gitea.io/gitea/models/activities"
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/models/organization"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
"code.gitea.io/gitea/models/unittest"
|
"code.gitea.io/gitea/models/unittest"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/json"
|
"code.gitea.io/gitea/modules/json"
|
||||||
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGetUserHeatmapDataByUser(t *testing.T) {
|
func TestGetUserHeatmapDataByUser(t *testing.T) {
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
|
||||||
|
// Mock time
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
insertHeatmapContribution(t, 2, 1, "user2-public-author-date", 1603009800)
|
||||||
|
insertHeatmapContribution(t, 2, 1, "user2-public-same-bucket", 1603009850)
|
||||||
|
insertHeatmapContribution(t, 2, 2, "user2-private-hidden", 1603010700)
|
||||||
|
insertHeatmapContribution(t, 2, 4, "user2-hidden-owner-hidden", 1603010700)
|
||||||
|
insertHeatmapContribution(t, 2, 1, "user2-future-hidden", timeutil.TimeStampNow()+900)
|
||||||
|
insertHeatmapContribution(t, 2, 999999, "user2-deleted-repo-hidden", 1603011600)
|
||||||
|
insertHeatmapContribution(t, 3, 1, "other-author-ignored", 1603010700)
|
||||||
|
setUserVisibility(t, 5, structs.VisibleTypePrivate)
|
||||||
|
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
desc string
|
desc string
|
||||||
userID int64
|
userID int64
|
||||||
@@ -25,73 +49,211 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
|
|||||||
JSONResult string
|
JSONResult string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"self looks at action in private repo",
|
"self sees public author-date contributions only",
|
||||||
2, 2, 1, `[{"timestamp":1603227600,"contributions":1}]`,
|
2, 2, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"admin looks at action in private repo",
|
"admin sees public author-date contributions only",
|
||||||
2, 1, 1, `[{"timestamp":1603227600,"contributions":1}]`,
|
2, 1, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"other user looks at action in private repo",
|
"other user sees public author-date contributions only",
|
||||||
2, 3, 0, `[]`,
|
2, 3, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"nobody looks at action in private repo",
|
"anonymous sees public author-date contributions only",
|
||||||
2, 0, 0, `[]`,
|
2, 0, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"collaborator looks at action in private repo",
|
"different author is not counted for target user",
|
||||||
16, 15, 1, `[{"timestamp":1603267200,"contributions":1}]`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no action action not performed by target user",
|
|
||||||
3, 3, 0, `[]`,
|
3, 3, 0, `[]`,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"multiple actions performed with two grouped together",
|
|
||||||
10, 10, 3, `[{"timestamp":1603009800,"contributions":1},{"timestamp":1603010700,"contributions":2}]`,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
// Prepare
|
|
||||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
|
||||||
|
|
||||||
// Mock time
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
|
||||||
|
|
||||||
|
var doer *user_model.User
|
||||||
|
if tc.doerID != 0 {
|
||||||
|
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
|
||||||
|
}
|
||||||
|
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.CountResult, countHeatmapContributions(heatmap), "testcase '%s'", tc.desc)
|
||||||
|
|
||||||
|
// Test JSON rendering
|
||||||
|
jsonData, err := json.Marshal(heatmap)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.JSONEq(t, tc.JSONResult, string(jsonData))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetUserHeatmapDataByUserUsesCurrentRepoVisibility(t *testing.T) {
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
|
||||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
defer timeutil.MockUnset()
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
for _, tc := range testCases {
|
insertHeatmapContribution(t, 2, 1, "user2-visibility-flip", 1603009800)
|
||||||
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
|
|
||||||
|
|
||||||
var doer *user_model.User
|
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
if tc.doerID != 0 {
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
|
require.NoError(t, err)
|
||||||
}
|
assertHeatmapJSON(t, heatmap, `[{
|
||||||
|
"timestamp": 1603009800,
|
||||||
|
"contributions": 1
|
||||||
|
}]`)
|
||||||
|
|
||||||
// get the action for comparison
|
setRepoPrivate(t, 1, true)
|
||||||
actions, count, err := activities_model.GetFeeds(t.Context(), activities_model.GetFeedsOptions{
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
RequestedUser: user,
|
require.NoError(t, err)
|
||||||
Actor: doer,
|
assert.Empty(t, heatmap)
|
||||||
IncludePrivate: true,
|
|
||||||
OnlyPerformedBy: true,
|
|
||||||
IncludeDeleted: true,
|
|
||||||
})
|
|
||||||
assert.NoError(t, err)
|
|
||||||
|
|
||||||
// Get the heatmap and compare
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, true))
|
||||||
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
var contributions int
|
require.NoError(t, err)
|
||||||
for _, hm := range heatmap {
|
assertHeatmapJSON(t, heatmap, `[{
|
||||||
contributions += int(hm.Contributions)
|
"timestamp": 1603009800,
|
||||||
}
|
"contributions": 1
|
||||||
assert.NoError(t, err)
|
}]`)
|
||||||
assert.Len(t, actions, contributions, "invalid action count: did the test data became too old?")
|
}
|
||||||
assert.Equal(t, count, int64(contributions))
|
|
||||||
assert.Equal(t, tc.CountResult, contributions, "testcase '%s'", tc.desc)
|
|
||||||
|
|
||||||
// Test JSON rendering
|
func TestGetUserHeatmapDataByOrgTeam(t *testing.T) {
|
||||||
jsonData, err := json.Marshal(heatmap)
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
assert.NoError(t, err)
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
assert.JSONEq(t, tc.JSONResult, string(jsonData))
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 3, false))
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
insertHeatmapContribution(t, 2, 32, "org-team-public", 1603009800)
|
||||||
|
insertHeatmapContribution(t, 2, 3, "org-team-private-hidden", 1603010700)
|
||||||
|
insertHeatmapContribution(t, 2, 1, "non-org-repo-ignored", 1603011600)
|
||||||
|
|
||||||
|
org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3})
|
||||||
|
team := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: 7})
|
||||||
|
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
|
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByOrgTeam(t.Context(), org, team, doer)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, countHeatmapContributions(heatmap))
|
||||||
|
|
||||||
|
jsonData, err := json.Marshal(heatmap)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.JSONEq(t, `[{"timestamp":1603009800,"contributions":1}]`, string(jsonData))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUserHeatmapPrivateContributionsOptIn(t *testing.T) {
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
insertHeatmapContribution(t, 16, 21, "user16-public", 1603009800)
|
||||||
|
insertHeatmapContribution(t, 16, 22, "user16-private-one", 1603009850)
|
||||||
|
insertHeatmapContribution(t, 16, 22, "user16-private-two", 1603010700)
|
||||||
|
|
||||||
|
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 16})
|
||||||
|
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||||
|
authenticated := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3})
|
||||||
|
collaborator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15})
|
||||||
|
|
||||||
|
for _, doer := range []*user_model.User{nil, target, admin, authenticated, collaborator} {
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, doer)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, countHeatmapContributions(heatmap), "private contributions should be hidden by default from %s", heatmapDoerName(doer))
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, true))
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assertHeatmapJSON(t, heatmap, `[{"timestamp":1603009800,"contributions":2},{"timestamp":1603010700,"contributions":1}]`)
|
||||||
|
|
||||||
|
target.KeepActivityPrivate = true
|
||||||
|
_, err = db.GetEngine(t.Context()).ID(target.ID).Cols("keep_activity_private").Update(target)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, heatmap)
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, authenticated)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, heatmap)
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, target)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 3, countHeatmapContributions(heatmap))
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, admin)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 3, countHeatmapContributions(heatmap))
|
||||||
|
}
|
||||||
|
|
||||||
|
func insertHeatmapContribution(t *testing.T, userID, repoID int64, commitSHA string, authorUnix timeutil.TimeStamp) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: fmt.Sprintf("%040x", sha1.Sum([]byte(commitSHA))),
|
||||||
|
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func setRepoPrivate(t *testing.T, repoID int64, isPrivate bool) {
|
||||||
|
t.Helper()
|
||||||
|
repo := &repo_model.Repository{ID: repoID, IsPrivate: isPrivate}
|
||||||
|
_, err := db.GetEngine(t.Context()).ID(repoID).Cols("is_private").Update(repo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setUserVisibility(t *testing.T, userID int64, visibility structs.VisibleType) {
|
||||||
|
t.Helper()
|
||||||
|
user := &user_model.User{ID: userID, Visibility: visibility}
|
||||||
|
_, err := db.GetEngine(t.Context()).ID(userID).Cols("visibility").Update(user)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertHeatmapJSON(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected string) {
|
||||||
|
t.Helper()
|
||||||
|
jsonData, err := json.Marshal(heatmap)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.JSONEq(t, expected, string(jsonData))
|
||||||
|
|
||||||
|
var decoded []map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(jsonData, &decoded))
|
||||||
|
for _, entry := range decoded {
|
||||||
|
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, mapKeys(entry))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mapKeys[K comparable, V any](m map[K]V) []K {
|
||||||
|
keys := make([]K, 0, len(m))
|
||||||
|
for key := range m {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
func countHeatmapContributions(heatmap []*activities_model.UserHeatmapData) int {
|
||||||
|
var contributions int
|
||||||
|
for _, hm := range heatmap {
|
||||||
|
contributions += int(hm.Contributions)
|
||||||
|
}
|
||||||
|
return contributions
|
||||||
|
}
|
||||||
|
|
||||||
|
func heatmapDoerName(doer *user_model.User) string {
|
||||||
|
if doer == nil {
|
||||||
|
return "anonymous"
|
||||||
|
}
|
||||||
|
return doer.Name
|
||||||
|
}
|
||||||
|
|||||||
@@ -405,6 +405,7 @@ func prepareMigrationTasks() []*migration {
|
|||||||
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
|
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
|
||||||
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
|
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
|
||||||
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
|
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
|
||||||
|
newMigration(331, "Create heatmap contribution table", v1_26.CreateHeatmapContributionTable),
|
||||||
}
|
}
|
||||||
return preparedMigrations
|
return preparedMigrations
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package v1_26
|
||||||
|
|
||||||
|
import (
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"xorm.io/xorm"
|
||||||
|
"xorm.io/xorm/schemas"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HeatmapContribution struct { //revive:disable-line:exported
|
||||||
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
UserID int64 `xorm:"NOT NULL"`
|
||||||
|
RepoID int64 `xorm:"NOT NULL"`
|
||||||
|
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
|
||||||
|
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
|
||||||
|
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
|
||||||
|
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||||
|
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TableIndices implements xorm's TableIndices interface.
|
||||||
|
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
|
||||||
|
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
|
||||||
|
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
|
||||||
|
|
||||||
|
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
|
||||||
|
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
|
||||||
|
|
||||||
|
return []*schemas.Index{uniqueContribution, userAuthorRepo}
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateHeatmapContributionTable(x *xorm.Engine) error {
|
||||||
|
_, err := x.SyncWithOptions(xorm.SyncOptions{IgnoreDropIndices: true}, new(HeatmapContribution))
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"code.gitea.io/gitea/models/db"
|
"code.gitea.io/gitea/models/db"
|
||||||
@@ -19,6 +20,8 @@ import (
|
|||||||
"xorm.io/xorm/convert"
|
"xorm.io/xorm/convert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const SettingsKeyIncludePrivateContributions = "include_private_contributions"
|
||||||
|
|
||||||
// Setting is a key value store of user settings
|
// Setting is a key value store of user settings
|
||||||
type Setting struct {
|
type Setting struct {
|
||||||
ID int64 `xorm:"pk autoincr"`
|
ID int64 `xorm:"pk autoincr"`
|
||||||
@@ -254,3 +257,17 @@ func SetUserSettingJSON[T any](ctx context.Context, userID int64, key string, va
|
|||||||
}
|
}
|
||||||
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
|
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetIncludePrivateContributions returns whether a user opted in to private heatmap contributions.
|
||||||
|
func GetIncludePrivateContributions(ctx context.Context, userID int64) (bool, error) {
|
||||||
|
value, err := GetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, "false")
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return strconv.ParseBool(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetIncludePrivateContributions stores whether a user opted in to private heatmap contributions.
|
||||||
|
func SetIncludePrivateContributions(ctx context.Context, userID int64, include bool) error {
|
||||||
|
return SetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, strconv.FormatBool(include))
|
||||||
|
}
|
||||||
|
|||||||
@@ -80,8 +80,9 @@ type UserSettings struct {
|
|||||||
Theme string `json:"theme"`
|
Theme string `json:"theme"`
|
||||||
DiffViewStyle string `json:"diff_view_style"`
|
DiffViewStyle string `json:"diff_view_style"`
|
||||||
// Privacy
|
// Privacy
|
||||||
HideEmail bool `json:"hide_email"`
|
HideEmail bool `json:"hide_email"`
|
||||||
HideActivity bool `json:"hide_activity"`
|
HideActivity bool `json:"hide_activity"`
|
||||||
|
IncludePrivateContributions bool `json:"include_private_contributions"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// UserSettingsOptions represents options to change user settings
|
// UserSettingsOptions represents options to change user settings
|
||||||
@@ -95,8 +96,9 @@ type UserSettingsOptions struct {
|
|||||||
Theme *string `json:"theme"`
|
Theme *string `json:"theme"`
|
||||||
DiffViewStyle *string `json:"diff_view_style"`
|
DiffViewStyle *string `json:"diff_view_style"`
|
||||||
// Privacy
|
// Privacy
|
||||||
HideEmail *bool `json:"hide_email"`
|
HideEmail *bool `json:"hide_email"`
|
||||||
HideActivity *bool `json:"hide_activity"`
|
HideActivity *bool `json:"hide_activity"`
|
||||||
|
IncludePrivateContributions *bool `json:"include_private_contributions"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenameUserOption options when renaming a user
|
// RenameUserOption options when renaming a user
|
||||||
|
|||||||
@@ -708,6 +708,8 @@
|
|||||||
"settings.privacy": "Privacy",
|
"settings.privacy": "Privacy",
|
||||||
"settings.keep_activity_private": "Hide Activity from profile page",
|
"settings.keep_activity_private": "Hide Activity from profile page",
|
||||||
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
|
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
|
||||||
|
"settings.include_private_contributions": "Show private contributions on profile heatmap",
|
||||||
|
"settings.include_private_contributions_popup": "Publicly reveals only contribution dates and counts from non-public repositories, never repository or commit details.",
|
||||||
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
|
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
|
||||||
"settings.federated_avatar_lookup": "Federated Avatar Lookup",
|
"settings.federated_avatar_lookup": "Federated Avatar Lookup",
|
||||||
"settings.enable_custom_avatar": "Use Custom Avatar",
|
"settings.enable_custom_avatar": "Use Custom Avatar",
|
||||||
|
|||||||
@@ -24,7 +24,12 @@ func GetUserSettings(ctx *context.APIContext) {
|
|||||||
// responses:
|
// responses:
|
||||||
// "200":
|
// "200":
|
||||||
// "$ref": "#/responses/UserSettings"
|
// "$ref": "#/responses/UserSettings"
|
||||||
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
|
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
|
||||||
|
if err != nil {
|
||||||
|
ctx.APIErrorInternal(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.JSON(http.StatusOK, settings)
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateUserSettings returns user settings
|
// UpdateUserSettings returns user settings
|
||||||
@@ -46,20 +51,26 @@ func UpdateUserSettings(ctx *context.APIContext) {
|
|||||||
form := web.GetForm(ctx).(*api.UserSettingsOptions)
|
form := web.GetForm(ctx).(*api.UserSettingsOptions)
|
||||||
|
|
||||||
opts := &user_service.UpdateOptions{
|
opts := &user_service.UpdateOptions{
|
||||||
FullName: optional.FromPtr(form.FullName),
|
FullName: optional.FromPtr(form.FullName),
|
||||||
Description: optional.FromPtr(form.Description),
|
Description: optional.FromPtr(form.Description),
|
||||||
Website: optional.FromPtr(form.Website),
|
Website: optional.FromPtr(form.Website),
|
||||||
Location: optional.FromPtr(form.Location),
|
Location: optional.FromPtr(form.Location),
|
||||||
Language: optional.FromPtr(form.Language),
|
Language: optional.FromPtr(form.Language),
|
||||||
Theme: optional.FromPtr(form.Theme),
|
Theme: optional.FromPtr(form.Theme),
|
||||||
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
|
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
|
||||||
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
|
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
|
||||||
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
|
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
|
||||||
|
IncludePrivateContributions: optional.FromPtr(form.IncludePrivateContributions),
|
||||||
}
|
}
|
||||||
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
|
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
|
||||||
ctx.APIErrorInternal(err)
|
ctx.APIErrorInternal(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
|
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
|
||||||
|
if err != nil {
|
||||||
|
ctx.APIErrorInternal(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.JSON(http.StatusOK, settings)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,16 +48,32 @@ func Profile(ctx *context.Context) {
|
|||||||
ctx.Data["PageIsSettingsProfile"] = true
|
ctx.Data["PageIsSettingsProfile"] = true
|
||||||
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
||||||
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
||||||
|
if !loadProfilePrivateContributionSetting(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadProfilePrivateContributionSetting(ctx *context.Context) bool {
|
||||||
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, ctx.Doer.ID)
|
||||||
|
if err != nil {
|
||||||
|
ctx.ServerError("GetIncludePrivateContributions", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ctx.Data["IncludePrivateContributions"] = includePrivateContributions
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// ProfilePost response for change user's profile
|
// ProfilePost response for change user's profile
|
||||||
func ProfilePost(ctx *context.Context) {
|
func ProfilePost(ctx *context.Context) {
|
||||||
ctx.Data["Title"] = ctx.Tr("settings_title")
|
ctx.Data["Title"] = ctx.Tr("settings_title")
|
||||||
ctx.Data["PageIsSettingsProfile"] = true
|
ctx.Data["PageIsSettingsProfile"] = true
|
||||||
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
||||||
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
||||||
|
if !loadProfilePrivateContributionSetting(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if ctx.HasError() {
|
if ctx.HasError() {
|
||||||
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
||||||
@@ -94,12 +110,13 @@ func ProfilePost(ctx *context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
opts := &user_service.UpdateOptions{
|
opts := &user_service.UpdateOptions{
|
||||||
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
|
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
|
||||||
Description: optional.Some(form.Description),
|
Description: optional.Some(form.Description),
|
||||||
Website: optional.Some(form.Website),
|
Website: optional.Some(form.Website),
|
||||||
Location: optional.Some(form.Location),
|
Location: optional.Some(form.Location),
|
||||||
Visibility: optional.Some(form.Visibility),
|
Visibility: optional.Some(form.Visibility),
|
||||||
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
|
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
|
||||||
|
IncludePrivateContributions: optional.Some(form.IncludePrivateContributions),
|
||||||
}
|
}
|
||||||
|
|
||||||
if form.FullName != "" {
|
if form.FullName != "" {
|
||||||
|
|||||||
@@ -86,18 +86,23 @@ func toUser(ctx context.Context, user *user_model.User, signed, authed bool) *ap
|
|||||||
}
|
}
|
||||||
|
|
||||||
// User2UserSettings return UserSettings based on a user
|
// User2UserSettings return UserSettings based on a user
|
||||||
func User2UserSettings(user *user_model.User) api.UserSettings {
|
func User2UserSettings(ctx context.Context, user *user_model.User) (api.UserSettings, error) {
|
||||||
return api.UserSettings{
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
|
||||||
FullName: user.FullName,
|
if err != nil {
|
||||||
Website: user.Website,
|
return api.UserSettings{}, err
|
||||||
Location: user.Location,
|
|
||||||
Language: user.Language,
|
|
||||||
Description: user.Description,
|
|
||||||
Theme: user.Theme,
|
|
||||||
HideEmail: user.KeepEmailPrivate,
|
|
||||||
HideActivity: user.KeepActivityPrivate,
|
|
||||||
DiffViewStyle: user.DiffViewStyle,
|
|
||||||
}
|
}
|
||||||
|
return api.UserSettings{
|
||||||
|
FullName: user.FullName,
|
||||||
|
Website: user.Website,
|
||||||
|
Location: user.Location,
|
||||||
|
Language: user.Language,
|
||||||
|
Description: user.Description,
|
||||||
|
Theme: user.Theme,
|
||||||
|
HideEmail: user.KeepEmailPrivate,
|
||||||
|
HideActivity: user.KeepActivityPrivate,
|
||||||
|
IncludePrivateContributions: includePrivateContributions,
|
||||||
|
DiffViewStyle: user.DiffViewStyle,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ToUserAndPermission return User and its collaboration permission for a repository
|
// ToUserAndPermission return User and its collaboration permission for a repository
|
||||||
|
|||||||
@@ -208,14 +208,15 @@ func (f *IntrospectTokenForm) Validate(req *http.Request, errs binding.Errors) b
|
|||||||
|
|
||||||
// UpdateProfileForm form for updating profile
|
// UpdateProfileForm form for updating profile
|
||||||
type UpdateProfileForm struct {
|
type UpdateProfileForm struct {
|
||||||
Name string `binding:"Username;MaxSize(40)"`
|
Name string `binding:"Username;MaxSize(40)"`
|
||||||
FullName string `binding:"MaxSize(100)"`
|
FullName string `binding:"MaxSize(100)"`
|
||||||
KeepEmailPrivate bool
|
KeepEmailPrivate bool
|
||||||
Website string `binding:"ValidSiteUrl;MaxSize(255)"`
|
Website string `binding:"ValidSiteUrl;MaxSize(255)"`
|
||||||
Location string `binding:"MaxSize(50)"`
|
Location string `binding:"MaxSize(50)"`
|
||||||
Description string `binding:"MaxSize(255)"`
|
Description string `binding:"MaxSize(255)"`
|
||||||
Visibility structs.VisibleType
|
Visibility structs.VisibleType
|
||||||
KeepActivityPrivate bool
|
KeepActivityPrivate bool
|
||||||
|
IncludePrivateContributions bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate validates the fields
|
// Validate validates the fields
|
||||||
|
|||||||
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
|||||||
defer gitRepo.Close()
|
defer gitRepo.Close()
|
||||||
|
|
||||||
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
|
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
|
||||||
|
indexHeatmap := false
|
||||||
if len(results) > 0 {
|
if len(results) > 0 {
|
||||||
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
|
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
|
||||||
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
|
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
|
||||||
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
|||||||
if result.RefName.IsPull() {
|
if result.RefName.IsPull() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
|
||||||
|
indexHeatmap = true
|
||||||
|
}
|
||||||
|
|
||||||
// Create reference
|
// Create reference
|
||||||
if result.OldCommitID == "" {
|
if result.OldCommitID == "" {
|
||||||
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
|||||||
NewCommitID: newCommitID,
|
NewCommitID: newCommitID,
|
||||||
}, theCommits)
|
}, theCommits)
|
||||||
}
|
}
|
||||||
|
if indexHeatmap {
|
||||||
|
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
|
||||||
|
log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
|
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
|
||||||
|
|
||||||
isEmpty, err := gitRepo.IsEmpty()
|
isEmpty, err := gitRepo.IsEmpty()
|
||||||
|
|||||||
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
|
|||||||
}
|
}
|
||||||
|
|
||||||
notify_service.ChangeDefaultBranch(ctx, repo)
|
notify_service.ChangeDefaultBranch(ctx, repo)
|
||||||
|
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return repo, nil
|
return repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/gitrepo"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
const heatmapIndexCommitsPageSize = 100
|
||||||
|
|
||||||
|
// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch.
|
||||||
|
func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error {
|
||||||
|
if repo == nil || repo.ID == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if repo.DefaultBranch == "" || repo.IsEmpty {
|
||||||
|
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer closer.Close()
|
||||||
|
|
||||||
|
head, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
|
||||||
|
if git.IsErrNotExist(err) {
|
||||||
|
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0)
|
||||||
|
now := timeutil.TimeStampNow()
|
||||||
|
for page := 1; ; page++ {
|
||||||
|
commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(commits) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, commit := range commits {
|
||||||
|
commitSHA := commit.ID.String()
|
||||||
|
if commit.Author == nil || commit.Author.Email == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
authorUnix := timeutil.TimeStamp(commit.Author.When.Unix())
|
||||||
|
if authorUnix > now {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repo.ID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: strings.ToLower(commit.Author.Email),
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{
|
||||||
|
RepoID: repo.ID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
UserID: userID,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(commits) < heatmapIndexCommitsPageSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) {
|
||||||
|
address, err := user_model.GetEmailAddressByEmail(ctx, email)
|
||||||
|
if user_model.IsErrEmailAddressNotExist(err) {
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, err
|
||||||
|
}
|
||||||
|
if !address.IsActivated {
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
return address.UID, true, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||||
|
repo_module "code.gitea.io/gitea/modules/repository"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
{Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
{Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
|
||||||
|
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, int64(2), contributions[0].UserID)
|
||||||
|
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
|
||||||
|
assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail)
|
||||||
|
assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix)
|
||||||
|
|
||||||
|
emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com")
|
||||||
|
require.NoError(t, err)
|
||||||
|
emailAddress.UID = 1
|
||||||
|
require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress))
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
|
||||||
|
contributions = loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, int64(1), contributions[0].UserID)
|
||||||
|
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
{Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
{Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
|
||||||
|
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 2)
|
||||||
|
assert.Equal(t, int64(2), contributions[0].UserID)
|
||||||
|
assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA)
|
||||||
|
assert.Equal(t, int64(1), contributions[1].UserID)
|
||||||
|
assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA)
|
||||||
|
for _, contribution := range contributions {
|
||||||
|
assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA)
|
||||||
|
}
|
||||||
|
|
||||||
|
deleteMainRef(t, repo)
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
|
||||||
|
|
||||||
|
newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
})
|
||||||
|
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
|
||||||
|
{
|
||||||
|
RefFullName: git.RefNameFromBranch("main"),
|
||||||
|
OldCommitID: commits["initial"],
|
||||||
|
NewCommitID: newCommits["pushed"],
|
||||||
|
PusherID: 1,
|
||||||
|
RepoUserName: repo.OwnerName,
|
||||||
|
RepoName: repo.Name,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 2)
|
||||||
|
assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
|
||||||
|
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
|
||||||
|
{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
|
||||||
|
})
|
||||||
|
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
|
||||||
|
{
|
||||||
|
RefFullName: git.RefNameFromBranch("feature"),
|
||||||
|
OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(),
|
||||||
|
NewCommitID: featureCommits["feature-pushed"],
|
||||||
|
PusherID: 1,
|
||||||
|
RepoUserName: repo.OwnerName,
|
||||||
|
RepoName: repo.Name,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
type heatmapIndexTestCommit struct {
|
||||||
|
Branch string
|
||||||
|
Mark string
|
||||||
|
Parent string
|
||||||
|
AuthorName string
|
||||||
|
AuthorEmail string
|
||||||
|
CommitterName string
|
||||||
|
CommitterEmail string
|
||||||
|
AuthorDate string
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
|
||||||
|
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
|
repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
marks := runFastImport(t, repo, commits)
|
||||||
|
repo.IsEmpty = false
|
||||||
|
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
|
||||||
|
return repo, marks
|
||||||
|
}
|
||||||
|
|
||||||
|
func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var stream strings.Builder
|
||||||
|
branchTips := make(map[string]string)
|
||||||
|
for i, commit := range commits {
|
||||||
|
mark := fmt.Sprintf(":%d", i+1)
|
||||||
|
branchRef := "refs/heads/" + commit.Branch
|
||||||
|
stream.WriteString("commit " + branchRef + "\n")
|
||||||
|
stream.WriteString("mark " + mark + "\n")
|
||||||
|
stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
|
||||||
|
stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
|
||||||
|
message := "heatmap " + commit.Mark
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
|
||||||
|
if parentMark := branchTips[commit.Branch]; parentMark != "" {
|
||||||
|
stream.WriteString("from " + parentMark + "\n")
|
||||||
|
} else if commit.Parent != "" {
|
||||||
|
stream.WriteString("from " + commit.Parent + "\n")
|
||||||
|
}
|
||||||
|
content := commit.Mark + "\n"
|
||||||
|
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
|
||||||
|
branchTips[commit.Branch] = mark
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
WithStdinCopy(strings.NewReader(stream.String())).
|
||||||
|
Run(t.Context()))
|
||||||
|
|
||||||
|
commitSHAs := make(map[string]string, len(commits))
|
||||||
|
for _, commit := range commits {
|
||||||
|
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
|
||||||
|
AddOptionFormat("--grep=%s", "heatmap "+commit.Mark).
|
||||||
|
AddDynamicArguments("refs/heads/" + commit.Branch).
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
RunStdString(t.Context())
|
||||||
|
require.NoError(t, err)
|
||||||
|
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
|
||||||
|
}
|
||||||
|
return commitSHAs
|
||||||
|
}
|
||||||
|
|
||||||
|
func signatureLine(kind, name, email, date string) string {
|
||||||
|
parsed, err := time.Parse(time.RFC3339, date)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
|
||||||
|
t.Helper()
|
||||||
|
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return contributions
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteMainRef(t *testing.T, repo *repo_model.Repository) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
|
||||||
|
}
|
||||||
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
|
repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
|
||||||
if opts.Mirror {
|
if opts.Mirror {
|
||||||
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
|
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
|||||||
}
|
}
|
||||||
return repo, nil
|
return repo, nil
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
|
||||||
|
}
|
||||||
|
return repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
|
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
indexHeatmap := false
|
||||||
if !opts.IsDelRef() {
|
if !opts.IsDelRef() {
|
||||||
branch := opts.RefFullName.BranchName()
|
branch := opts.RefFullName.BranchName()
|
||||||
|
|
||||||
@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
|||||||
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
|
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
|
||||||
log.Error("DelRepoDivergenceFromCache: %v", err)
|
log.Error("DelRepoDivergenceFromCache: %v", err)
|
||||||
}
|
}
|
||||||
|
indexHeatmap = true
|
||||||
} else {
|
} else {
|
||||||
if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
|
if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
|
||||||
log.Error("DelDivergenceFromCache: %v", err)
|
log.Error("DelDivergenceFromCache: %v", err)
|
||||||
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
|||||||
pushDeleteBranch(ctx, repo, pusher, opts)
|
pushDeleteBranch(ctx, repo, pusher, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if indexHeatmap {
|
||||||
|
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
||||||
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
|
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
|
||||||
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
|
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
auth_model "code.gitea.io/gitea/models/auth"
|
auth_model "code.gitea.io/gitea/models/auth"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
password_module "code.gitea.io/gitea/modules/auth/password"
|
password_module "code.gitea.io/gitea/modules/auth/password"
|
||||||
"code.gitea.io/gitea/modules/optional"
|
"code.gitea.io/gitea/modules/optional"
|
||||||
@@ -47,6 +48,7 @@ type UpdateOptions struct {
|
|||||||
IsRestricted optional.Option[bool]
|
IsRestricted optional.Option[bool]
|
||||||
Visibility optional.Option[structs.VisibleType]
|
Visibility optional.Option[structs.VisibleType]
|
||||||
KeepActivityPrivate optional.Option[bool]
|
KeepActivityPrivate optional.Option[bool]
|
||||||
|
IncludePrivateContributions optional.Option[bool]
|
||||||
Language optional.Option[string]
|
Language optional.Option[string]
|
||||||
Theme optional.Option[string]
|
Theme optional.Option[string]
|
||||||
DiffViewStyle optional.Option[string]
|
DiffViewStyle optional.Option[string]
|
||||||
@@ -182,7 +184,23 @@ func UpdateUser(ctx context.Context, u *user_model.User, opts *UpdateOptions) er
|
|||||||
cols = append(cols, "last_login_unix")
|
cols = append(cols, "last_login_unix")
|
||||||
}
|
}
|
||||||
|
|
||||||
return user_model.UpdateUserCols(ctx, u, cols...)
|
if len(cols) > 0 || opts.IncludePrivateContributions.Has() {
|
||||||
|
return db.WithTx(ctx, func(ctx context.Context) error {
|
||||||
|
if len(cols) > 0 {
|
||||||
|
if err := user_model.UpdateUserCols(ctx, u, cols...); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.IncludePrivateContributions.Has() {
|
||||||
|
return user_model.SetIncludePrivateContributions(ctx, u.ID, opts.IncludePrivateContributions.Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type UpdateAuthOptions struct {
|
type UpdateAuthOptions struct {
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ func TestUpdateUser(t *testing.T) {
|
|||||||
IsAdmin: UpdateOptionFieldFromValue(true),
|
IsAdmin: UpdateOptionFieldFromValue(true),
|
||||||
Visibility: optional.Some(structs.VisibleTypePrivate),
|
Visibility: optional.Some(structs.VisibleTypePrivate),
|
||||||
KeepActivityPrivate: optional.Some(true),
|
KeepActivityPrivate: optional.Some(true),
|
||||||
|
IncludePrivateContributions: optional.Some(true),
|
||||||
Language: optional.Some("lang"),
|
Language: optional.Some("lang"),
|
||||||
Theme: optional.Some("theme"),
|
Theme: optional.Some("theme"),
|
||||||
DiffViewStyle: optional.Some("split"),
|
DiffViewStyle: optional.Some("split"),
|
||||||
@@ -66,6 +67,9 @@ func TestUpdateUser(t *testing.T) {
|
|||||||
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
||||||
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
||||||
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
||||||
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
|
||||||
assert.Equal(t, opts.Language.Value(), user.Language)
|
assert.Equal(t, opts.Language.Value(), user.Language)
|
||||||
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
||||||
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
||||||
@@ -86,6 +90,9 @@ func TestUpdateUser(t *testing.T) {
|
|||||||
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
||||||
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
||||||
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
||||||
|
includePrivateContributions, err = user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
|
||||||
assert.Equal(t, opts.Language.Value(), user.Language)
|
assert.Equal(t, opts.Language.Value(), user.Language)
|
||||||
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
||||||
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
||||||
|
|||||||
+9
-1
@@ -29655,6 +29655,10 @@
|
|||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
"x-go-name": "HideEmail"
|
"x-go-name": "HideEmail"
|
||||||
},
|
},
|
||||||
|
"include_private_contributions": {
|
||||||
|
"type": "boolean",
|
||||||
|
"x-go-name": "IncludePrivateContributions"
|
||||||
|
},
|
||||||
"language": {
|
"language": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"x-go-name": "Language"
|
"x-go-name": "Language"
|
||||||
@@ -29699,6 +29703,10 @@
|
|||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
"x-go-name": "HideEmail"
|
"x-go-name": "HideEmail"
|
||||||
},
|
},
|
||||||
|
"include_private_contributions": {
|
||||||
|
"type": "boolean",
|
||||||
|
"x-go-name": "IncludePrivateContributions"
|
||||||
|
},
|
||||||
"language": {
|
"language": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"x-go-name": "Language"
|
"x-go-name": "Language"
|
||||||
@@ -31073,4 +31081,4 @@
|
|||||||
"TOTPHeader": []
|
"TOTPHeader": []
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,6 +88,13 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="field">
|
||||||
|
<div class="ui checkbox" id="include-private-contributions">
|
||||||
|
<label data-tooltip-content="{{ctx.Locale.Tr "settings.include_private_contributions_popup"}}"><strong>{{ctx.Locale.Tr "settings.include_private_contributions"}}</strong></label>
|
||||||
|
<input name="include_private_contributions" type="checkbox" {{if .IncludePrivateContributions}}checked{{end}}>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="divider"></div>
|
<div class="divider"></div>
|
||||||
|
|
||||||
<div class="field">
|
<div class="field">
|
||||||
|
|||||||
@@ -4,17 +4,19 @@
|
|||||||
package integration
|
package integration
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
activities_model "code.gitea.io/gitea/models/activities"
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
auth_model "code.gitea.io/gitea/models/auth"
|
auth_model "code.gitea.io/gitea/models/auth"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
"code.gitea.io/gitea/tests"
|
"code.gitea.io/gitea/tests"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestUserHeatmap(t *testing.T) {
|
func TestUserHeatmap(t *testing.T) {
|
||||||
@@ -23,17 +25,23 @@ func TestUserHeatmap(t *testing.T) {
|
|||||||
normalUsername := "user2"
|
normalUsername := "user2"
|
||||||
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
|
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
|
||||||
|
|
||||||
fakeNow := time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local)
|
fakeNow := time.Date(2011, 10, 21, 0, 0, 0, 0, time.Local)
|
||||||
timeutil.MockSet(fakeNow)
|
timeutil.MockSet(fakeNow)
|
||||||
defer timeutil.MockUnset()
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)).
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-author-date", 1319068800)
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-same-bucket", 1319068850)
|
||||||
|
testHeatmapSeedContribution(t, 2, 2, "api-user2-private-hidden", 1319070600)
|
||||||
|
|
||||||
|
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", normalUsername).
|
||||||
AddTokenAuth(token)
|
AddTokenAuth(token)
|
||||||
resp := MakeRequest(t, req, http.StatusOK)
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
var heatmap []*activities_model.UserHeatmapData
|
var heatmap []*activities_model.UserHeatmapData
|
||||||
DecodeJSON(t, resp, &heatmap)
|
DecodeJSON(t, resp, &heatmap)
|
||||||
var dummyheatmap []*activities_model.UserHeatmapData
|
|
||||||
dummyheatmap = append(dummyheatmap, &activities_model.UserHeatmapData{Timestamp: 1603227600, Contributions: 1})
|
|
||||||
|
|
||||||
assert.Equal(t, dummyheatmap, heatmap)
|
assert.Equal(t, []*activities_model.UserHeatmapData{
|
||||||
|
{Timestamp: 1319068800, Contributions: 2},
|
||||||
|
}, heatmap)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,22 +4,40 @@
|
|||||||
package integration
|
package integration
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha1"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/json"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
"code.gitea.io/gitea/tests"
|
"code.gitea.io/gitea/tests"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type testHeatmapWebResponse struct {
|
||||||
|
HeatmapData [][2]int64 `json:"heatmapData"`
|
||||||
|
TotalContributions int64 `json:"totalContributions"`
|
||||||
|
}
|
||||||
|
|
||||||
func TestHeatmapEndpoints(t *testing.T) {
|
func TestHeatmapEndpoints(t *testing.T) {
|
||||||
defer tests.PrepareTestEnv(t)()
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
// Mock time so fixture actions fall within the heatmap's time window
|
// Mock time so fixture actions fall within the heatmap's time window
|
||||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
defer timeutil.MockUnset()
|
defer timeutil.MockUnset()
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-one", 1603009800)
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-two", 1603009850)
|
||||||
|
testHeatmapSeedContribution(t, 2, 2, "web-user2-private-hidden", 1603010700)
|
||||||
|
|
||||||
session := loginUser(t, "user2")
|
session := loginUser(t, "user2")
|
||||||
|
|
||||||
@@ -28,11 +46,15 @@ func TestHeatmapEndpoints(t *testing.T) {
|
|||||||
req := NewRequest(t, "GET", "/user2/-/heatmap")
|
req := NewRequest(t, "GET", "/user2/-/heatmap")
|
||||||
resp := session.MakeRequest(t, req, http.StatusOK)
|
resp := session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
|
||||||
var result map[string]any
|
webHeatmap := testHeatmapDecodeWebResponse(t, resp)
|
||||||
DecodeJSON(t, resp, &result)
|
|
||||||
assert.Contains(t, result, "heatmapData")
|
req = NewRequest(t, "GET", "/api/v1/users/user2/heatmap")
|
||||||
assert.Contains(t, result, "totalContributions")
|
resp = session.MakeRequest(t, req, http.StatusOK)
|
||||||
assert.Positive(t, result["totalContributions"])
|
var apiHeatmap []*activities_model.UserHeatmapData
|
||||||
|
DecodeJSON(t, resp, &apiHeatmap)
|
||||||
|
|
||||||
|
assert.Equal(t, testHeatmapSumAPIContributions(apiHeatmap), webHeatmap.TotalContributions)
|
||||||
|
assert.Equal(t, int64(2), webHeatmap.TotalContributions)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("OrgDashboard", func(t *testing.T) {
|
t.Run("OrgDashboard", func(t *testing.T) {
|
||||||
@@ -57,3 +79,71 @@ func TestHeatmapEndpoints(t *testing.T) {
|
|||||||
assert.Contains(t, result, "totalContributions")
|
assert.Contains(t, result, "totalContributions")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testHeatmapSeedContribution(t *testing.T, userID, repoID int64, label string, authorUnix timeutil.TimeStamp) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
commitSHA := fmt.Sprintf("%040x", sha1.Sum([]byte(label)))
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}))
|
||||||
|
return commitSHA
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapDecodeWebResponse(t *testing.T, resp *httptest.ResponseRecorder) testHeatmapWebResponse {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var result testHeatmapWebResponse
|
||||||
|
DecodeJSON(t, resp, &result)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapSumAPIContributions(heatmap []*activities_model.UserHeatmapData) int64 {
|
||||||
|
var total int64
|
||||||
|
for _, item := range heatmap {
|
||||||
|
total += item.Contributions
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapAssertAggregateOnlyAPIResponse(t *testing.T, body []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var decoded []map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(body, &decoded))
|
||||||
|
for _, entry := range decoded {
|
||||||
|
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, testHeatmapMapKeys(entry))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapAssertAggregateOnlyWebResponse(t *testing.T, body []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(body, &decoded))
|
||||||
|
assert.ElementsMatch(t, []string{"heatmapData", "totalContributions"}, testHeatmapMapKeys(decoded))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapAssertNoPrivateMetadata(t *testing.T, body []byte, forbidden ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
response := string(body)
|
||||||
|
for _, value := range forbidden {
|
||||||
|
assert.NotContains(t, response, value)
|
||||||
|
}
|
||||||
|
for _, value := range []string{"repo_id", "repository_id", "repoID", "commit_sha", "commit_message", "branch", "url", "action_id"} {
|
||||||
|
assert.NotContains(t, response, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapMapKeys[K comparable, V any](m map[K]V) []K {
|
||||||
|
keys := make([]K, 0, len(m))
|
||||||
|
for key := range m {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,16 +7,20 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
activities_model "code.gitea.io/gitea/models/activities"
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
auth_model "code.gitea.io/gitea/models/auth"
|
auth_model "code.gitea.io/gitea/models/auth"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
repo_model "code.gitea.io/gitea/models/repo"
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
"code.gitea.io/gitea/models/unittest"
|
"code.gitea.io/gitea/models/unittest"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
api "code.gitea.io/gitea/modules/structs"
|
api "code.gitea.io/gitea/modules/structs"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
"code.gitea.io/gitea/tests"
|
"code.gitea.io/gitea/tests"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -24,6 +28,8 @@ const (
|
|||||||
privateActivityTestUser = "user2"
|
privateActivityTestUser = "user2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const privateActivityHeatmapOptInUser = "user16"
|
||||||
|
|
||||||
// org3 is an organization so it is not usable here
|
// org3 is an organization so it is not usable here
|
||||||
const privateActivityTestOtherUser = "user4"
|
const privateActivityTestOtherUser = "user4"
|
||||||
|
|
||||||
@@ -41,6 +47,7 @@ func testPrivateActivityDoSomethingForActionEntries(t *testing.T) {
|
|||||||
Title: "test",
|
Title: "test",
|
||||||
}).AddTokenAuth(token)
|
}).AddTokenAuth(token)
|
||||||
session.MakeRequest(t, req, http.StatusCreated)
|
session.MakeRequest(t, req, http.StatusCreated)
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "private-activity-action-public", timeutil.TimeStampNow()-900)
|
||||||
}
|
}
|
||||||
|
|
||||||
// private activity helpers
|
// private activity helpers
|
||||||
@@ -56,6 +63,26 @@ func testPrivateActivityHelperEnablePrivateActivity(t *testing.T) {
|
|||||||
session.MakeRequest(t, req, http.StatusSeeOther)
|
session.MakeRequest(t, req, http.StatusSeeOther)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t *testing.T, session *TestSession, username string, include bool) {
|
||||||
|
values := map[string]string{
|
||||||
|
"name": username,
|
||||||
|
"email": username + "@example.com",
|
||||||
|
"language": "en-US",
|
||||||
|
}
|
||||||
|
if include {
|
||||||
|
values["include_private_contributions"] = "1"
|
||||||
|
}
|
||||||
|
req := NewRequestWithValues(t, "POST", "/user/settings", values)
|
||||||
|
session.MakeRequest(t, req, http.StatusSeeOther)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperAssertIncludePrivateContributions(t *testing.T, username string, expected bool) {
|
||||||
|
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: username})
|
||||||
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expected, includePrivateContributions)
|
||||||
|
}
|
||||||
|
|
||||||
func testPrivateActivityHelperHasVisibleActivitiesInHTMLDoc(htmlDoc *HTMLDoc) bool {
|
func testPrivateActivityHelperHasVisibleActivitiesInHTMLDoc(htmlDoc *HTMLDoc) bool {
|
||||||
return htmlDoc.doc.Find("#activity-feed").Find(".flex-item").Length() > 0
|
return htmlDoc.doc.Find("#activity-feed").Find(".flex-item").Length() > 0
|
||||||
}
|
}
|
||||||
@@ -136,6 +163,196 @@ func testPrivateActivityHelperHasHeatmapContentFromSession(t *testing.T, session
|
|||||||
return len(items) != 0
|
return len(items) != 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperGetAPIHeatmapFromPublic(t *testing.T) ([]*activities_model.UserHeatmapData, []byte) {
|
||||||
|
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", privateActivityHeatmapOptInUser)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
body := resp.Body.Bytes()
|
||||||
|
|
||||||
|
var items []*activities_model.UserHeatmapData
|
||||||
|
DecodeJSON(t, resp, &items)
|
||||||
|
|
||||||
|
return items, body
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperGetAPIHeatmapFromSession(t *testing.T, session *TestSession) ([]*activities_model.UserHeatmapData, []byte) {
|
||||||
|
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeReadUser)
|
||||||
|
|
||||||
|
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", privateActivityHeatmapOptInUser).
|
||||||
|
AddTokenAuth(token)
|
||||||
|
resp := session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
body := resp.Body.Bytes()
|
||||||
|
|
||||||
|
var items []*activities_model.UserHeatmapData
|
||||||
|
DecodeJSON(t, resp, &items)
|
||||||
|
|
||||||
|
return items, body
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperGetWebHeatmapFromPublic(t *testing.T) (testHeatmapWebResponse, []byte) {
|
||||||
|
req := NewRequestf(t, "GET", "/%s/-/heatmap", privateActivityHeatmapOptInUser)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
body := resp.Body.Bytes()
|
||||||
|
|
||||||
|
return testHeatmapDecodeWebResponse(t, resp), body
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperSeedHeatmapContributions(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
|
||||||
|
|
||||||
|
testHeatmapSeedContribution(t, 16, 21, "private-activity-public", 1603009800)
|
||||||
|
privateSHA := testHeatmapSeedContribution(t, 16, 22, "private-activity-private-one", 1603009850)
|
||||||
|
testHeatmapSeedContribution(t, 16, 22, "private-activity-private-two", 1603010700)
|
||||||
|
|
||||||
|
return privateSHA
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPrivateActivityHelperAssertAPIHeatmapTotal(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected int64) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
assert.Equal(t, expected, testHeatmapSumAPIContributions(heatmap))
|
||||||
|
}
|
||||||
|
|
||||||
|
// check private contribution opt-in settings persistence and ownership
|
||||||
|
|
||||||
|
func TestPrivateActivityIncludePrivateContributionsWebSettingsPersistence(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
session := loginUser(t, privateActivityTestUser)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
|
||||||
|
|
||||||
|
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, session, privateActivityTestUser, true)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, true)
|
||||||
|
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: privateActivityTestUser})
|
||||||
|
assert.False(t, user.KeepActivityPrivate, "private contribution opt-in must not hide the whole activity heatmap")
|
||||||
|
|
||||||
|
req := NewRequest(t, "GET", "/user/settings")
|
||||||
|
resp := session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
htmlDoc := NewHTMLParser(t, resp.Body)
|
||||||
|
assert.Equal(t, 1, htmlDoc.doc.Find("#include-private-contributions input[name='include_private_contributions']:checked").Length())
|
||||||
|
|
||||||
|
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, session, privateActivityTestUser, false)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrivateActivityIncludePrivateContributionsAPISettingsPersistence(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
session := loginUser(t, privateActivityTestUser)
|
||||||
|
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeReadUser, auth_model.AccessTokenScopeWriteUser)
|
||||||
|
|
||||||
|
includePrivateContributions := true
|
||||||
|
req := NewRequestWithJSON(t, "PATCH", "/api/v1/user/settings", &api.UserSettingsOptions{
|
||||||
|
IncludePrivateContributions: &includePrivateContributions,
|
||||||
|
}).AddTokenAuth(token)
|
||||||
|
resp := session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
settings := DecodeJSON(t, resp, &api.UserSettings{})
|
||||||
|
assert.True(t, settings.IncludePrivateContributions)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, true)
|
||||||
|
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: privateActivityTestUser})
|
||||||
|
assert.False(t, user.KeepActivityPrivate, "API opt-in must not set KeepActivityPrivate")
|
||||||
|
|
||||||
|
req = NewRequest(t, "GET", "/api/v1/user/settings").AddTokenAuth(token)
|
||||||
|
resp = session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
settings = DecodeJSON(t, resp, &api.UserSettings{})
|
||||||
|
assert.True(t, settings.IncludePrivateContributions)
|
||||||
|
|
||||||
|
includePrivateContributions = false
|
||||||
|
req = NewRequestWithJSON(t, "PATCH", "/api/v1/user/settings", &api.UserSettingsOptions{
|
||||||
|
IncludePrivateContributions: &includePrivateContributions,
|
||||||
|
}).AddTokenAuth(token)
|
||||||
|
resp = session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
settings = DecodeJSON(t, resp, &api.UserSettings{})
|
||||||
|
assert.False(t, settings.IncludePrivateContributions)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrivateActivityPrivateContributionsHiddenByDefault(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
testPrivateActivityHelperSeedHeatmapContributions(t)
|
||||||
|
|
||||||
|
publicHeatmap, _ := testPrivateActivityHelperGetAPIHeatmapFromPublic(t)
|
||||||
|
testPrivateActivityHelperAssertAPIHeatmapTotal(t, publicHeatmap, 1)
|
||||||
|
|
||||||
|
for _, testCase := range []struct {
|
||||||
|
name string
|
||||||
|
username string
|
||||||
|
}{
|
||||||
|
{"owner", privateActivityHeatmapOptInUser},
|
||||||
|
{"admin", privateActivityTestAdmin},
|
||||||
|
{"collaborator", "user15"},
|
||||||
|
{"unrelated", privateActivityTestOtherUser},
|
||||||
|
} {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
session := loginUser(t, testCase.username)
|
||||||
|
heatmap, _ := testPrivateActivityHelperGetAPIHeatmapFromSession(t, session)
|
||||||
|
testPrivateActivityHelperAssertAPIHeatmapTotal(t, heatmap, 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrivateActivityIncludePrivateContributionsEndpointAggregatesOnly(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
privateSHA := testPrivateActivityHelperSeedHeatmapContributions(t)
|
||||||
|
|
||||||
|
ownerSession := loginUser(t, privateActivityHeatmapOptInUser)
|
||||||
|
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, ownerSession, privateActivityHeatmapOptInUser, true)
|
||||||
|
|
||||||
|
apiHeatmap, apiBody := testPrivateActivityHelperGetAPIHeatmapFromPublic(t)
|
||||||
|
testPrivateActivityHelperAssertAPIHeatmapTotal(t, apiHeatmap, 3)
|
||||||
|
testHeatmapAssertAggregateOnlyAPIResponse(t, apiBody)
|
||||||
|
testHeatmapAssertNoPrivateMetadata(t, apiBody,
|
||||||
|
"big_test_private_3",
|
||||||
|
"22",
|
||||||
|
privateSHA,
|
||||||
|
"private-activity-private-one",
|
||||||
|
"master",
|
||||||
|
"/user16/big_test_private_3",
|
||||||
|
)
|
||||||
|
|
||||||
|
webHeatmap, webBody := testPrivateActivityHelperGetWebHeatmapFromPublic(t)
|
||||||
|
assert.Equal(t, int64(3), webHeatmap.TotalContributions)
|
||||||
|
testHeatmapAssertAggregateOnlyWebResponse(t, webBody)
|
||||||
|
testHeatmapAssertNoPrivateMetadata(t, webBody,
|
||||||
|
"big_test_private_3",
|
||||||
|
"22",
|
||||||
|
privateSHA,
|
||||||
|
"private-activity-private-one",
|
||||||
|
"master",
|
||||||
|
"/user16/big_test_private_3",
|
||||||
|
)
|
||||||
|
|
||||||
|
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, ownerSession, privateActivityHeatmapOptInUser, false)
|
||||||
|
apiHeatmap, _ = testPrivateActivityHelperGetAPIHeatmapFromPublic(t)
|
||||||
|
testPrivateActivityHelperAssertAPIHeatmapTotal(t, apiHeatmap, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrivateActivityIncludePrivateContributionsWebSettingsAuthBoundary(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
|
||||||
|
|
||||||
|
otherSession := loginUser(t, privateActivityTestOtherUser)
|
||||||
|
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, otherSession, privateActivityTestOtherUser, true)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestOtherUser, true)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
|
||||||
|
|
||||||
|
req := NewRequestWithValues(t, "POST", "/user/settings", map[string]string{
|
||||||
|
"name": privateActivityTestUser,
|
||||||
|
"email": privateActivityTestUser + "@example.com",
|
||||||
|
"language": "en-US",
|
||||||
|
"include_private_contributions": "1",
|
||||||
|
})
|
||||||
|
MakeRequest(t, req, http.StatusSeeOther)
|
||||||
|
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
|
||||||
|
}
|
||||||
|
|
||||||
// check activity visibility if the visibility is enabled
|
// check activity visibility if the visibility is enabled
|
||||||
|
|
||||||
func TestPrivateActivityNoVisibleForPublic(t *testing.T) {
|
func TestPrivateActivityNoVisibleForPublic(t *testing.T) {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
{ dash, hectic, sqlite, postgresql_17, gawk, self }:
|
{ dash, hectic, sqlite, postgresql_17, gawk, coreutils, self }:
|
||||||
let
|
let
|
||||||
shell = "${dash}/bin/dash";
|
shell = "${dash}/bin/dash";
|
||||||
bashOptions = [
|
bashOptions = [
|
||||||
@@ -11,7 +11,7 @@ let
|
|||||||
migrator = hectic.writeShellApplication {
|
migrator = hectic.writeShellApplication {
|
||||||
inherit shell bashOptions;
|
inherit shell bashOptions;
|
||||||
name = "migrator";
|
name = "migrator";
|
||||||
runtimeInputs = [ sqlite postgresql_17 gawk ];
|
runtimeInputs = [ sqlite postgresql_17 gawk coreutils ];
|
||||||
|
|
||||||
text = ''
|
text = ''
|
||||||
${builtins.readFile hectic.helpers.posix-shell.log}
|
${builtins.readFile hectic.helpers.posix-shell.log}
|
||||||
|
|||||||
@@ -154,11 +154,11 @@ init() {
|
|||||||
shift 2
|
shift 2
|
||||||
;;
|
;;
|
||||||
--*|-*)
|
--*|-*)
|
||||||
printf 'init argument %s does not exists' "$1"
|
log error "init argument ${WHITE}$1${NC} does not exists"
|
||||||
exit 9
|
exit 9
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
printf 'init command %s does not exists' "$1"
|
log error "init subcommand ${WHITE}$1${NC} does not exists"
|
||||||
exit 9
|
exit 9
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
@@ -434,6 +434,8 @@ migrate_down() {
|
|||||||
target_migration=$(printf '%s' "$fs_migrations" | sed -n "${target_line}p")
|
target_migration=$(printf '%s' "$fs_migrations" | sed -n "${target_line}p")
|
||||||
printf '%s' "$target_migration"
|
printf '%s' "$target_migration"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
migrate_up() {
|
migrate_up() {
|
||||||
@@ -488,6 +490,7 @@ migrate_up() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
printf '%s' "$target_migration"
|
printf '%s' "$target_migration"
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
migrate_to() {
|
migrate_to() {
|
||||||
@@ -518,6 +521,8 @@ migrate_to() {
|
|||||||
printf '%s' "$migration_name"
|
printf '%s' "$migration_name"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
migration_list() {
|
migration_list() {
|
||||||
@@ -566,7 +571,10 @@ index_of() {
|
|||||||
|
|
||||||
# no subshell, no pipeline
|
# no subshell, no pipeline
|
||||||
while IFS= read -r m; do
|
while IFS= read -r m; do
|
||||||
[ "$m" = "$name" ] && { printf '%s\n' "$i"; return 0; }
|
if [ "$m" = "$name" ]; then
|
||||||
|
printf '%s\n' "$i"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
i=$((i+1))
|
i=$((i+1))
|
||||||
done <<EOF
|
done <<EOF
|
||||||
$list
|
$list
|
||||||
@@ -633,7 +641,8 @@ migrate() {
|
|||||||
db_mig_count=$(printf '%s' "$db_migrations" | wc -l)
|
db_mig_count=$(printf '%s' "$db_migrations" | wc -l)
|
||||||
log debug "mig count: $db_mig_count"
|
log debug "mig count: $db_mig_count"
|
||||||
|
|
||||||
# Log migration lists for debugging
|
# These counts are useful even at normal verbosity because they summarize the
|
||||||
|
# migration state before any validation or apply/revert work begins.
|
||||||
fs_mig_count=$(printf '%s' "$fs_migrations" | wc -l)
|
fs_mig_count=$(printf '%s' "$fs_migrations" | wc -l)
|
||||||
log info "Filesystem migrations found: ${WHITE}$fs_mig_count"
|
log info "Filesystem migrations found: ${WHITE}$fs_mig_count"
|
||||||
log info "Database migrations applied: ${WHITE}$db_mig_count"
|
log info "Database migrations applied: ${WHITE}$db_mig_count"
|
||||||
@@ -696,26 +705,35 @@ migrate() {
|
|||||||
log info "Migration history validation: ${GREEN}OK${NC} (${WHITE}$i${NC} migrations match)"
|
log info "Migration history validation: ${GREEN}OK${NC} (${WHITE}$i${NC} migrations match)"
|
||||||
|
|
||||||
eval "set -- $MIGRATOR_REMAINING_ARS"
|
eval "set -- $MIGRATOR_REMAINING_ARS"
|
||||||
target_migration="$("migrate_$MIGRATE_SUBCOMMAND" "$@")"
|
if ! target_migration="$("migrate_$MIGRATE_SUBCOMMAND" "$@")"; then
|
||||||
|
log error "failed to resolve migration target for ${WHITE}migrate $MIGRATE_SUBCOMMAND${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [ -z "$db_migrations" ]; then
|
if [ -z "$db_migrations" ]; then
|
||||||
log info "it'll firs migration"
|
log info "starting from clean migration state"
|
||||||
current_idx=0
|
current_idx=0
|
||||||
else
|
else
|
||||||
current_migration=$(printf '%s\n' "$db_migrations" | tail -n1)
|
current_migration=$(printf '%s\n' "$db_migrations" | tail -n1)
|
||||||
current_idx=$(index_of "$fs_migrations" "$current_migration")
|
if ! current_idx=$(index_of "$fs_migrations" "$current_migration"); then
|
||||||
|
log error "current database migration is missing from filesystem: ${WHITE}$current_migration${NC}"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log debug "[$WHITE$fs_migrations$NC]"
|
log debug "filesystem migrations: ${WHITE}$fs_migrations${NC}"
|
||||||
log debug "$target_migration"
|
log debug "requested target migration: ${WHITE}${target_migration:-<clean state>}${NC}"
|
||||||
|
|
||||||
if [ -z "$target_migration" ]; then
|
if [ -z "$target_migration" ]; then
|
||||||
target_idx=0
|
target_idx=0
|
||||||
else
|
else
|
||||||
target_idx=$(index_of "$fs_migrations" "$target_migration")
|
if ! target_idx=$(index_of "$fs_migrations" "$target_migration"); then
|
||||||
|
log error "target migration is missing from filesystem: ${WHITE}$target_migration${NC}"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log debug "indexes $WHITE$current_idx$NC $WHITE${target_idx}"
|
log debug "migration indexes: current=${WHITE}$current_idx${NC} target=${WHITE}$target_idx${NC}"
|
||||||
|
|
||||||
if [ "$target_idx" -eq "$current_idx" ]; then
|
if [ "$target_idx" -eq "$current_idx" ]; then
|
||||||
if [ "$target_idx" -eq 0 ]; then
|
if [ "$target_idx" -eq 0 ]; then
|
||||||
@@ -903,11 +921,11 @@ list() {
|
|||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
--*|-*)
|
--*|-*)
|
||||||
log error "init argument $1 does not exists"
|
log error "list argument ${WHITE}$1${NC} does not exists"
|
||||||
exit 9
|
exit 9
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
log error "init subcommand $1 does not exists"
|
log error "list subcommand ${WHITE}$1${NC} does not exists"
|
||||||
exit 9
|
exit 9
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
buildGoModule,
|
||||||
|
fetchFromGitHub,
|
||||||
|
lib,
|
||||||
|
}:
|
||||||
|
buildGoModule {
|
||||||
|
pname = "plgo";
|
||||||
|
version = "0.2.1-unstable-2026-04-04";
|
||||||
|
|
||||||
|
src = fetchFromGitHub {
|
||||||
|
owner = "mgr43";
|
||||||
|
repo = "plgo";
|
||||||
|
rev = "492abd81f50f5f588794bd3f9fa103d14546b275";
|
||||||
|
hash = "sha256-Ifd2IFA62VtkqCUP0/cD3jT4SqkjqwCpHVolslQbwkA=";
|
||||||
|
};
|
||||||
|
|
||||||
|
vendorHash = "sha256-KIVHepJRfVa0bkTF1A4g6ZFGxDy8OPzY3iwV3MhRyVA=";
|
||||||
|
|
||||||
|
proxyVendor = true;
|
||||||
|
|
||||||
|
subPackages = [ "cmd/plgo" ];
|
||||||
|
|
||||||
|
doCheck = false;
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "Write PostgreSQL extensions in Go";
|
||||||
|
homepage = "https://github.com/mgr43/plgo";
|
||||||
|
license = lib.licenses.bsd3;
|
||||||
|
mainProgram = "plgo";
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Local HTTP API for Diffusers Stable Video Diffusion image-to-video."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import uuid
|
||||||
|
from importlib import import_module
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
torch = import_module("torch")
|
||||||
|
uvicorn = import_module("uvicorn")
|
||||||
|
diffusers = import_module("diffusers")
|
||||||
|
diffusers_utils = import_module("diffusers.utils")
|
||||||
|
fastapi = import_module("fastapi")
|
||||||
|
pil_image = import_module("PIL.Image")
|
||||||
|
pil_unidentified_image_error = import_module("PIL").UnidentifiedImageError
|
||||||
|
|
||||||
|
FastAPI = fastapi.FastAPI
|
||||||
|
File = fastapi.File
|
||||||
|
Form = fastapi.Form
|
||||||
|
HTTPException = fastapi.HTTPException
|
||||||
|
Request = fastapi.Request
|
||||||
|
UploadFile = fastapi.UploadFile
|
||||||
|
StableVideoDiffusionPipeline = diffusers.StableVideoDiffusionPipeline
|
||||||
|
export_to_video = diffusers_utils.export_to_video
|
||||||
|
|
||||||
|
|
||||||
|
HOST = os.environ.get("SVD_API_HOST", "127.0.0.1")
|
||||||
|
PORT = int(os.environ.get("SVD_API_PORT", "8000"))
|
||||||
|
MODEL_ID = os.environ.get("SVD_MODEL_ID", "stabilityai/stable-video-diffusion-img2vid-xt")
|
||||||
|
CACHE_DIR = os.environ.get("SVD_CACHE_DIR") or os.environ.get("HF_HOME")
|
||||||
|
OUTPUT_DIR = Path(os.environ.get("SVD_OUTPUT_DIR", "/var/lib/stable-video-diffusion-api/outputs"))
|
||||||
|
DEVICE = os.environ.get("SVD_DEVICE", "cuda" if torch.cuda.is_available() else "cpu")
|
||||||
|
DTYPE = os.environ.get("SVD_DTYPE", "float16")
|
||||||
|
ENABLE_CPU_OFFLOAD = os.environ.get("SVD_ENABLE_CPU_OFFLOAD", "0").lower() in ("1", "true", "yes", "on")
|
||||||
|
DEFAULT_DECODE_CHUNK_SIZE = int(os.environ.get("SVD_DECODE_CHUNK_SIZE", "8"))
|
||||||
|
DEFAULT_FPS = int(os.environ.get("SVD_FPS", "7"))
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(title="Stable Video Diffusion API")
|
||||||
|
pipeline_lock = threading.Lock()
|
||||||
|
pipeline: Any | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def torch_dtype() -> Any:
|
||||||
|
dtypes = {
|
||||||
|
"float16": torch.float16,
|
||||||
|
"fp16": torch.float16,
|
||||||
|
"float32": torch.float32,
|
||||||
|
"fp32": torch.float32,
|
||||||
|
"bfloat16": torch.bfloat16,
|
||||||
|
"bf16": torch.bfloat16,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
return dtypes[DTYPE.lower()]
|
||||||
|
except KeyError as exc:
|
||||||
|
raise RuntimeError(f"Unsupported SVD_DTYPE: {DTYPE}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def get_pipeline() -> Any:
|
||||||
|
global pipeline
|
||||||
|
|
||||||
|
if pipeline is not None:
|
||||||
|
return pipeline
|
||||||
|
|
||||||
|
with pipeline_lock:
|
||||||
|
if pipeline is not None:
|
||||||
|
return pipeline
|
||||||
|
|
||||||
|
kwargs: dict[str, Any] = {"torch_dtype": torch_dtype()}
|
||||||
|
if CACHE_DIR:
|
||||||
|
kwargs["cache_dir"] = CACHE_DIR
|
||||||
|
|
||||||
|
loaded = StableVideoDiffusionPipeline.from_pretrained(MODEL_ID, **kwargs)
|
||||||
|
if ENABLE_CPU_OFFLOAD:
|
||||||
|
loaded.enable_model_cpu_offload()
|
||||||
|
else:
|
||||||
|
loaded.to(DEVICE)
|
||||||
|
|
||||||
|
pipeline = loaded
|
||||||
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
|
def positive_int(name: str, value: Any, default: int) -> int:
|
||||||
|
if value in (None, ""):
|
||||||
|
return default
|
||||||
|
try:
|
||||||
|
parsed = int(value)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=f"{name} must be an integer") from exc
|
||||||
|
if parsed <= 0:
|
||||||
|
raise HTTPException(status_code=400, detail=f"{name} must be greater than zero")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def optional_int(name: str, value: Any) -> int | None:
|
||||||
|
if value in (None, ""):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=f"{name} must be an integer") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def optional_float(name: str, value: Any) -> float | None:
|
||||||
|
if value in (None, ""):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=f"{name} must be a number") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def load_image_from_path(input_image_path: str) -> Any:
|
||||||
|
path = Path(input_image_path).expanduser()
|
||||||
|
if not path.exists() or not path.is_file():
|
||||||
|
raise HTTPException(status_code=400, detail="input_image_path does not exist or is not a file")
|
||||||
|
try:
|
||||||
|
return pil_image.open(path).convert("RGB")
|
||||||
|
except (OSError, pil_unidentified_image_error) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail="input_image_path could not be decoded as an image") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def load_uploaded_image(image: UploadFile) -> Any:
|
||||||
|
try:
|
||||||
|
return pil_image.open(image.file).convert("RGB")
|
||||||
|
except (OSError, pil_unidentified_image_error) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail="uploaded image could not be decoded") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def json_payload(request: Request) -> dict[str, Any]:
|
||||||
|
content_type = request.headers.get("content-type", "")
|
||||||
|
if not content_type.startswith("application/json"):
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
payload = await request.json()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail="request body must be valid JSON") from exc
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise HTTPException(status_code=400, detail="JSON request body must be an object")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health() -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"model_loaded": pipeline is not None,
|
||||||
|
"model_id": MODEL_ID,
|
||||||
|
"device": DEVICE,
|
||||||
|
"dtype": DTYPE,
|
||||||
|
"cpu_offload": ENABLE_CPU_OFFLOAD,
|
||||||
|
"cache_dir": CACHE_DIR,
|
||||||
|
"output_dir": str(OUTPUT_DIR),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/generate")
|
||||||
|
async def generate(
|
||||||
|
request: Request,
|
||||||
|
image: UploadFile | None = File(default=None),
|
||||||
|
input_image_path: str | None = Form(default=None),
|
||||||
|
num_frames: int | None = Form(default=None),
|
||||||
|
num_inference_steps: int | None = Form(default=None),
|
||||||
|
fps: int | None = Form(default=None),
|
||||||
|
decode_chunk_size: int | None = Form(default=None),
|
||||||
|
seed: int | None = Form(default=None),
|
||||||
|
motion_bucket_id: int | None = Form(default=None),
|
||||||
|
noise_aug_strength: float | None = Form(default=None),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
payload = await json_payload(request)
|
||||||
|
path = input_image_path or payload.get("input_image_path")
|
||||||
|
upload = image
|
||||||
|
|
||||||
|
if upload is None and not path:
|
||||||
|
raise HTTPException(status_code=400, detail="provide input_image_path or upload image")
|
||||||
|
if upload is not None and path:
|
||||||
|
raise HTTPException(status_code=400, detail="provide only one image source")
|
||||||
|
|
||||||
|
source_image = await load_uploaded_image(upload) if upload is not None else load_image_from_path(str(path))
|
||||||
|
chunk_size = positive_int("decode_chunk_size", decode_chunk_size if decode_chunk_size is not None else payload.get("decode_chunk_size"), DEFAULT_DECODE_CHUNK_SIZE)
|
||||||
|
output_fps = positive_int("fps", fps if fps is not None else payload.get("fps"), DEFAULT_FPS)
|
||||||
|
request_seed = optional_int("seed", seed if seed is not None else payload.get("seed"))
|
||||||
|
generator = None
|
||||||
|
if request_seed is not None:
|
||||||
|
generator = torch.Generator(device="cpu").manual_seed(request_seed)
|
||||||
|
|
||||||
|
call_args: dict[str, Any] = {
|
||||||
|
"image": source_image,
|
||||||
|
"decode_chunk_size": chunk_size,
|
||||||
|
}
|
||||||
|
for key, value in {
|
||||||
|
"num_frames": num_frames if num_frames is not None else payload.get("num_frames"),
|
||||||
|
"num_inference_steps": num_inference_steps if num_inference_steps is not None else payload.get("num_inference_steps"),
|
||||||
|
"motion_bucket_id": motion_bucket_id if motion_bucket_id is not None else payload.get("motion_bucket_id"),
|
||||||
|
}.items():
|
||||||
|
parsed = optional_int(key, value)
|
||||||
|
if parsed is not None:
|
||||||
|
call_args[key] = parsed
|
||||||
|
|
||||||
|
parsed_noise = optional_float("noise_aug_strength", noise_aug_strength if noise_aug_strength is not None else payload.get("noise_aug_strength"))
|
||||||
|
if parsed_noise is not None:
|
||||||
|
call_args["noise_aug_strength"] = parsed_noise
|
||||||
|
if generator is not None:
|
||||||
|
call_args["generator"] = generator
|
||||||
|
|
||||||
|
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
output_path = OUTPUT_DIR / f"svd-{uuid.uuid4().hex}.mp4"
|
||||||
|
|
||||||
|
try:
|
||||||
|
frames = get_pipeline()(**call_args).frames[0]
|
||||||
|
export_to_video(frames, str(output_path), fps=output_fps)
|
||||||
|
except Exception as exc:
|
||||||
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
return {
|
||||||
|
"output_path": str(output_path),
|
||||||
|
"model_id": MODEL_ID,
|
||||||
|
"seed": request_seed,
|
||||||
|
"fps": output_fps,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
uvicorn.run("app:app", host=HOST, port=PORT)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{ pkgs }:
|
||||||
|
|
||||||
|
let
|
||||||
|
pythonEnv = pkgs.python3.withPackages (ps: let
|
||||||
|
torch = ps.torchWithCuda;
|
||||||
|
accelerate = ps.accelerate.override { inherit torch; };
|
||||||
|
in [
|
||||||
|
accelerate
|
||||||
|
ps.diffusers
|
||||||
|
ps.fastapi
|
||||||
|
ps.imageio
|
||||||
|
ps.imageio-ffmpeg
|
||||||
|
ps.pillow
|
||||||
|
ps.python-multipart
|
||||||
|
ps.safetensors
|
||||||
|
torch
|
||||||
|
ps.transformers
|
||||||
|
ps.uvicorn
|
||||||
|
]);
|
||||||
|
in
|
||||||
|
|
||||||
|
pkgs.stdenv.mkDerivation {
|
||||||
|
pname = "stable-video-diffusion-api";
|
||||||
|
version = "0.1.0";
|
||||||
|
src = ./.;
|
||||||
|
|
||||||
|
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p $out/bin $out/libexec/stable-video-diffusion-api
|
||||||
|
cp $src/app.py $out/libexec/stable-video-diffusion-api/app.py
|
||||||
|
chmod +x $out/libexec/stable-video-diffusion-api/app.py
|
||||||
|
|
||||||
|
makeWrapper ${pythonEnv}/bin/python3 $out/bin/stable-video-diffusion-api \
|
||||||
|
--add-flags $out/libexec/stable-video-diffusion-api/app.py \
|
||||||
|
--set-default SVD_API_HOST 127.0.0.1 \
|
||||||
|
--set-default SVD_API_PORT 8000 \
|
||||||
|
--set-default SVD_MODEL_ID stabilityai/stable-video-diffusion-img2vid-xt \
|
||||||
|
--set-default SVD_OUTPUT_DIR /var/lib/stable-video-diffusion-api/outputs
|
||||||
|
'';
|
||||||
|
}
|
||||||
Generated
+1939
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
|||||||
|
[package]
|
||||||
|
name = "which-country-rs"
|
||||||
|
version = "1.0.0"
|
||||||
|
edition = "2024"
|
||||||
|
description = "CLI tool that detects your country by IP and renders an ASCII map"
|
||||||
|
license = "MIT"
|
||||||
|
repository = "https://github.com/krisfur/which-country-rs"
|
||||||
|
keywords = ["cli", "geolocation", "ascii", "map"]
|
||||||
|
categories = ["command-line-utilities"]
|
||||||
|
|
||||||
|
[features]
|
||||||
|
default = ["geoip"]
|
||||||
|
geoip = ["reqwest"]
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
clap = { version = "4", features = ["derive"] }
|
||||||
|
reqwest = { version = "0.13", features = ["blocking", "json"], optional = true }
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_json = "1"
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 Krzysztof Furman
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# which-country-rs
|
||||||
|
|
||||||
|
A CLI tool that detects your country from your IP address and renders an ASCII map zoomed into it, showing neighbouring borders and country codes.
|
||||||
|
|
||||||
|
## Example output
|
||||||
|
|
||||||
|
```bash
|
||||||
|
which-country-rs -c DE
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
You appear to be in: Germany (DE)
|
||||||
|
|
||||||
|
·········· ·· ······ · ·
|
||||||
|
· NO · ·· · ···
|
||||||
|
· · ···· ··· ···········
|
||||||
|
· ····· ·· ············
|
||||||
|
····· ········ · ·· ···EE ·
|
||||||
|
· ······ ······ ·· · LV····· ·····
|
||||||
|
·· ··· ·DK ·· ··· ···· ··········· ···
|
||||||
|
···· ··· ··· · · ··· ···· ··· LT ···· ···
|
||||||
|
· ····· ···· ··· ############ ··················· ···
|
||||||
|
IE ·· ····GB · ·····##### ## ## ···· BY
|
||||||
|
· ·· ·· ··· ·NL # ## ··
|
||||||
|
··· ····· ·· ····· ## # PL ·············
|
||||||
|
····················BE ··# DE #######····· ····
|
||||||
|
· ······· ···L## ### CZ ····· ······
|
||||||
|
······· #### ###·········SK ······· ··
|
||||||
|
····· ·###### ###### AT ········ ···············
|
||||||
|
·· FR ··· CH ··#·············· HU ·· MD··
|
||||||
|
· · ····· · ····SI··· ········· RO ····
|
||||||
|
· · · ···HR········ RS···· ···
|
||||||
|
········ ·· · ········ IT ··· ·····BA ·· ··············
|
||||||
|
· ·· ········· ·········· ·· · · ···ME··XK···· ··
|
||||||
|
····· ··· · ·· · ··· ···· ·· ··MK·· ·BG ·····
|
||||||
|
· ·· ····· ··· ····· ···· ·AL············ ······
|
||||||
|
· ·· ES · · · ·· ····· ···· ···· ······
|
||||||
|
|
||||||
|
Coordinates: 51.15°N, 10.55°E
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```
|
||||||
|
# Auto-detect from IP
|
||||||
|
which-country-rs
|
||||||
|
|
||||||
|
# Specify a country code
|
||||||
|
which-country-rs -c JP
|
||||||
|
|
||||||
|
# Specify coordinates (supports negative values for south/west)
|
||||||
|
which-country-rs --lat 40 --lon -74
|
||||||
|
|
||||||
|
# Custom map size
|
||||||
|
which-country-rs -W 120 -H 40
|
||||||
|
```
|
||||||
|
|
||||||
|
### Options
|
||||||
|
|
||||||
|
| Flag | Description | Default |
|
||||||
|
|------|-------------|---------|
|
||||||
|
| `-W, --width` | Map width in characters | 80 |
|
||||||
|
| `-H, --height` | Map height in characters | 24 |
|
||||||
|
| `-c, --country` | ISO 3166-1 alpha-2 country code (skips IP lookup) | |
|
||||||
|
| `--lat` | Latitude (requires `--lon`) | |
|
||||||
|
| `--lon` | Longitude (requires `--lat`) | |
|
||||||
|
| `-V, --version` | Print version | |
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
```
|
||||||
|
cargo build --release
|
||||||
|
```
|
||||||
|
|
||||||
|
To build without IP geolocation support (drops the `reqwest` dependency):
|
||||||
|
|
||||||
|
```
|
||||||
|
cargo build --release --no-default-features
|
||||||
|
```
|
||||||
|
|
||||||
|
## Map data
|
||||||
|
|
||||||
|
Country borders from [Natural Earth](https://www.naturalearthdata.com/) 110m admin-0 countries (public domain).
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,25 @@
|
|||||||
|
{ lib, pkgs, ... }: let
|
||||||
|
src = ./.;
|
||||||
|
in
|
||||||
|
pkgs.rustPlatform.buildRustPackage {
|
||||||
|
pname = "which-country-rs";
|
||||||
|
version = "1.0.0";
|
||||||
|
|
||||||
|
inherit src;
|
||||||
|
|
||||||
|
cargoLock.lockFile = ./Cargo.lock;
|
||||||
|
|
||||||
|
# NOTE(yukkop): upstream uses edition 2024, so do not pass the repo's
|
||||||
|
# Rust 1.81 commonArgs here.
|
||||||
|
# NOTE(yukkop): keep the Nix package offline-by-default; upstream's
|
||||||
|
# default geoip feature calls a plaintext HTTP endpoint.
|
||||||
|
cargoBuildFlags = ["--no-default-features"];
|
||||||
|
cargoTestFlags = ["--no-default-features"];
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "CLI tool that tells you which country contains coordinates";
|
||||||
|
homepage = "https://github.com/krisfur/which-country-rs";
|
||||||
|
license = lib.licenses.mit;
|
||||||
|
mainProgram = "which-country-rs";
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct FeatureCollection {
|
||||||
|
features: Vec<Feature>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct Feature {
|
||||||
|
properties: Properties,
|
||||||
|
geometry: Geometry,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct Properties {
|
||||||
|
#[serde(rename = "ISO_A2_EH")]
|
||||||
|
iso_a2: String,
|
||||||
|
#[serde(rename = "NAME")]
|
||||||
|
name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
enum Geometry {
|
||||||
|
Polygon {
|
||||||
|
coordinates: Vec<Vec<[f64; 2]>>,
|
||||||
|
},
|
||||||
|
MultiPolygon {
|
||||||
|
coordinates: Vec<Vec<Vec<[f64; 2]>>>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub struct Country {
|
||||||
|
pub iso_a2: String,
|
||||||
|
pub name: String,
|
||||||
|
/// Each polygon is a list of rings; ring 0 = outer, rest = holes
|
||||||
|
pub polygons: Vec<Vec<Vec<[f64; 2]>>>,
|
||||||
|
pub bbox: (f64, f64, f64, f64), // (min_lon, min_lat, max_lon, max_lat)
|
||||||
|
pub label_pos: (f64, f64), // (lon, lat) — centroid of largest polygon
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Signed area of a ring (positive = CCW).
|
||||||
|
fn ring_signed_area(ring: &[[f64; 2]]) -> f64 {
|
||||||
|
let n = ring.len();
|
||||||
|
if n < 3 {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
let mut area = 0.0;
|
||||||
|
let mut j = n - 1;
|
||||||
|
for i in 0..n {
|
||||||
|
area += (ring[j][0] - ring[i][0]) * (ring[j][1] + ring[i][1]);
|
||||||
|
j = i;
|
||||||
|
}
|
||||||
|
area / 2.0
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ray-casting point-in-ring test.
|
||||||
|
fn point_in_ring(lon: f64, lat: f64, ring: &[[f64; 2]]) -> bool {
|
||||||
|
let mut inside = false;
|
||||||
|
let n = ring.len();
|
||||||
|
if n < 3 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let mut j = n - 1;
|
||||||
|
for i in 0..n {
|
||||||
|
let xi = ring[i][0];
|
||||||
|
let yi = ring[i][1];
|
||||||
|
let xj = ring[j][0];
|
||||||
|
let yj = ring[j][1];
|
||||||
|
if ((yi > lat) != (yj > lat)) && (lon < (xj - xi) * (lat - yi) / (yj - yi) + xi) {
|
||||||
|
inside = !inside;
|
||||||
|
}
|
||||||
|
j = i;
|
||||||
|
}
|
||||||
|
inside
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Find horizontal interior spans at a given latitude.
|
||||||
|
/// Returns sorted pairs of (enter_lon, exit_lon).
|
||||||
|
fn horizontal_spans(lat: f64, ring: &[[f64; 2]]) -> Vec<(f64, f64)> {
|
||||||
|
let n = ring.len();
|
||||||
|
if n < 3 {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
let mut crossings = Vec::new();
|
||||||
|
let mut j = n - 1;
|
||||||
|
for i in 0..n {
|
||||||
|
let yi = ring[i][1];
|
||||||
|
let yj = ring[j][1];
|
||||||
|
if (yi > lat) != (yj > lat) {
|
||||||
|
let xi = ring[i][0];
|
||||||
|
let xj = ring[j][0];
|
||||||
|
crossings.push((xj - xi) * (lat - yi) / (yj - yi) + xi);
|
||||||
|
}
|
||||||
|
j = i;
|
||||||
|
}
|
||||||
|
crossings.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
crossings.chunks_exact(2).map(|p| (p[0], p[1])).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Find vertical interior spans at a given longitude.
|
||||||
|
/// Returns sorted pairs of (enter_lat, exit_lat).
|
||||||
|
fn vertical_spans(lon: f64, ring: &[[f64; 2]]) -> Vec<(f64, f64)> {
|
||||||
|
let n = ring.len();
|
||||||
|
if n < 3 {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
let mut crossings = Vec::new();
|
||||||
|
let mut j = n - 1;
|
||||||
|
for i in 0..n {
|
||||||
|
let xi = ring[i][0];
|
||||||
|
let xj = ring[j][0];
|
||||||
|
if (xi > lon) != (xj > lon) {
|
||||||
|
let yi = ring[i][1];
|
||||||
|
let yj = ring[j][1];
|
||||||
|
crossings.push((yj - yi) * (lon - xi) / (xj - xi) + yi);
|
||||||
|
}
|
||||||
|
j = i;
|
||||||
|
}
|
||||||
|
crossings.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
crossings.chunks_exact(2).map(|p| (p[0], p[1])).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Find a good interior label point for a ring.
|
||||||
|
/// Scans a grid of candidate points and picks the one that maximizes
|
||||||
|
/// min(half_width, half_height) — the "most interior" point.
|
||||||
|
fn ring_label_point(ring: &[[f64; 2]]) -> (f64, f64) {
|
||||||
|
let min_lon = ring.iter().map(|c| c[0]).fold(f64::MAX, f64::min);
|
||||||
|
let max_lon = ring.iter().map(|c| c[0]).fold(f64::MIN, f64::max);
|
||||||
|
let min_lat = ring.iter().map(|c| c[1]).fold(f64::MAX, f64::min);
|
||||||
|
let max_lat = ring.iter().map(|c| c[1]).fold(f64::MIN, f64::max);
|
||||||
|
|
||||||
|
let steps = 24;
|
||||||
|
let mut best = ((min_lon + max_lon) / 2.0, (min_lat + max_lat) / 2.0);
|
||||||
|
let mut best_score = 0.0f64;
|
||||||
|
|
||||||
|
for row in 1..steps {
|
||||||
|
let lat = min_lat + (max_lat - min_lat) * row as f64 / steps as f64;
|
||||||
|
let h_spans = horizontal_spans(lat, ring);
|
||||||
|
|
||||||
|
for &(span_left, span_right) in &h_spans {
|
||||||
|
let mid_lon = (span_left + span_right) / 2.0;
|
||||||
|
let half_w = (span_right - span_left) / 2.0;
|
||||||
|
|
||||||
|
// Measure vertical extent at this longitude
|
||||||
|
let v_spans = vertical_spans(mid_lon, ring);
|
||||||
|
for &(span_bot, span_top) in &v_spans {
|
||||||
|
if lat >= span_bot && lat <= span_top {
|
||||||
|
let half_h = ((lat - span_bot).min(span_top - lat)).min(half_w);
|
||||||
|
let score = half_w.min(half_h);
|
||||||
|
if score > best_score {
|
||||||
|
best_score = score;
|
||||||
|
best = (mid_lon, lat);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
best
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn load_countries(geojson: &str) -> Vec<Country> {
|
||||||
|
let fc: FeatureCollection = serde_json::from_str(geojson).expect("Failed to parse GeoJSON");
|
||||||
|
|
||||||
|
fc.features
|
||||||
|
.into_iter()
|
||||||
|
.map(|f| {
|
||||||
|
let polygons = match f.geometry {
|
||||||
|
Geometry::Polygon { coordinates } => vec![coordinates],
|
||||||
|
Geometry::MultiPolygon { coordinates } => coordinates,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut min_lon = f64::MAX;
|
||||||
|
let mut min_lat = f64::MAX;
|
||||||
|
let mut max_lon = f64::MIN;
|
||||||
|
let mut max_lat = f64::MIN;
|
||||||
|
|
||||||
|
for poly in &polygons {
|
||||||
|
for ring in poly {
|
||||||
|
for coord in ring {
|
||||||
|
let lon = coord[0];
|
||||||
|
let lat = coord[1];
|
||||||
|
if lon < min_lon {
|
||||||
|
min_lon = lon;
|
||||||
|
}
|
||||||
|
if lon > max_lon {
|
||||||
|
max_lon = lon;
|
||||||
|
}
|
||||||
|
if lat < min_lat {
|
||||||
|
min_lat = lat;
|
||||||
|
}
|
||||||
|
if lat > max_lat {
|
||||||
|
max_lat = lat;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Label inside the largest polygon
|
||||||
|
let label_pos = polygons
|
||||||
|
.iter()
|
||||||
|
.filter(|p| !p.is_empty() && p[0].len() >= 3)
|
||||||
|
.max_by(|a, b| {
|
||||||
|
ring_signed_area(&a[0])
|
||||||
|
.abs()
|
||||||
|
.partial_cmp(&ring_signed_area(&b[0]).abs())
|
||||||
|
.unwrap()
|
||||||
|
})
|
||||||
|
.map(|p| ring_label_point(&p[0]))
|
||||||
|
.unwrap_or(((min_lon + max_lon) / 2.0, (min_lat + max_lat) / 2.0));
|
||||||
|
|
||||||
|
Country {
|
||||||
|
iso_a2: f.properties.iso_a2,
|
||||||
|
name: f.properties.name,
|
||||||
|
polygons,
|
||||||
|
bbox: (min_lon, min_lat, max_lon, max_lat),
|
||||||
|
label_pos,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if a point is inside a polygon (outer ring minus holes).
|
||||||
|
fn point_in_polygon(lon: f64, lat: f64, rings: &[Vec<[f64; 2]>]) -> bool {
|
||||||
|
if rings.is_empty() || !point_in_ring(lon, lat, &rings[0]) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// Must be outside all holes
|
||||||
|
!rings[1..].iter().any(|hole| point_in_ring(lon, lat, hole))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if a point falls inside a country.
|
||||||
|
pub fn point_in_country(lon: f64, lat: f64, country: &Country) -> bool {
|
||||||
|
let (min_lon, min_lat, max_lon, max_lat) = country.bbox;
|
||||||
|
if lon < min_lon || lon > max_lon || lat < min_lat || lat > max_lat {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
country
|
||||||
|
.polygons
|
||||||
|
.iter()
|
||||||
|
.any(|poly| point_in_polygon(lon, lat, poly))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Find which country contains the given point, with a nearest-country
|
||||||
|
/// fallback for when low-res coastlines cause a near miss.
|
||||||
|
pub fn find_country(lon: f64, lat: f64, countries: &[Country]) -> Option<usize> {
|
||||||
|
// Exact hit
|
||||||
|
if let Some(idx) = countries.iter().position(|c| point_in_country(lon, lat, c)) {
|
||||||
|
return Some(idx);
|
||||||
|
}
|
||||||
|
// Search in expanding rings up to ~1 degree
|
||||||
|
for &offset in &[0.25, 0.5, 1.0] {
|
||||||
|
for &(dlon, dlat) in &[
|
||||||
|
(offset, 0.0),
|
||||||
|
(-offset, 0.0),
|
||||||
|
(0.0, offset),
|
||||||
|
(0.0, -offset),
|
||||||
|
(offset, offset),
|
||||||
|
(offset, -offset),
|
||||||
|
(-offset, offset),
|
||||||
|
(-offset, -offset),
|
||||||
|
] {
|
||||||
|
if let Some(idx) = countries
|
||||||
|
.iter()
|
||||||
|
.position(|c| point_in_country(lon + dlon, lat + dlat, c))
|
||||||
|
{
|
||||||
|
return Some(idx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(tag = "status")]
|
||||||
|
pub enum GeoIpResponse {
|
||||||
|
#[serde(rename = "success")]
|
||||||
|
Success {
|
||||||
|
country: String,
|
||||||
|
#[serde(rename = "countryCode")]
|
||||||
|
country_code: String,
|
||||||
|
lat: f64,
|
||||||
|
lon: f64,
|
||||||
|
},
|
||||||
|
#[serde(rename = "fail")]
|
||||||
|
Fail { message: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct GeoIpResult {
|
||||||
|
pub country: String,
|
||||||
|
pub country_code: String,
|
||||||
|
pub lat: f64,
|
||||||
|
pub lon: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn lookup() -> Result<GeoIpResult, String> {
|
||||||
|
let url = "http://ip-api.com/json/?fields=status,message,countryCode,country,lat,lon";
|
||||||
|
let resp = reqwest::blocking::get(url).map_err(|e| format!("HTTP request failed: {e}"))?;
|
||||||
|
let data: GeoIpResponse = resp.json().map_err(|e| format!("Failed to parse response: {e}"))?;
|
||||||
|
|
||||||
|
match data {
|
||||||
|
GeoIpResponse::Success {
|
||||||
|
country,
|
||||||
|
country_code,
|
||||||
|
lat,
|
||||||
|
lon,
|
||||||
|
} => Ok(GeoIpResult {
|
||||||
|
country,
|
||||||
|
country_code,
|
||||||
|
lat,
|
||||||
|
lon,
|
||||||
|
}),
|
||||||
|
GeoIpResponse::Fail { message } => Err(format!("GeoIP lookup failed: {message}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user