Compare commits

..

38 Commits

Author SHA1 Message Date
yukkop 557b6e9ef0 fix: migrator 2026-07-24 15:21:19 +04:00
yukkop e49f497045 fix: aga 2026-07-24 15:21:11 +04:00
yukkop feb1a48db1 feat: some 2026-07-23 14:19:58 +04:00
yukkop 30732080b7 feat: some 2026-07-21 16:57:10 +04:00
yukkop 80cf1588bb feat: +darwin config for yukkop
runner nix smoke / nix label smoke (push) Has been cancelled
2026-07-19 20:03:18 +04:00
yukkop ef7d1b29f4 feat: +njalla module 2026-07-13 23:37:14 +00:00
yukkop 7e8c6884db feat: +plgo 2026-07-13 13:27:04 +00:00
yukkop 1dd41e608b feat: neuro: stable video diffusion 2026-07-05 16:58:47 +00:00
yukkop e41c3e5a05 ssh fixes 2026-07-04 19:32:08 +00:00
yukkop f473280bf5 fix: matrix hardcode 2026-07-01 09:44:43 +00:00
yukkop b08fdd6e6b fix: matrix media 2026-07-01 09:23:26 +00:00
yukkop f73bfc63be fix: zomro: reboot 2026-06-20 22:00:52 +00:00
yukkop f6e7c1eca9 chore 2026-06-11 14:39:57 +00:00
yukkop 6996d178ef fix: hetzner: newer servers generation 2026-06-10 15:39:05 +00:00
yukkop 7f7229b199 feat: tenix host
runner nix smoke / nix label smoke (push) Has been cancelled
2026-06-10 13:15:09 +00:00
yukkop 2d5bd26c36 docs: migrator: ~ logs & comments 2026-06-10 12:26:16 +00:00
yukkop fba150b55b docs: db-tool: ~ postgres-init and postgres-cleanup 2026-06-10 12:14:23 +00:00
yukkop 2e7bf58acf refactor: db-tool: rename file 2026-06-10 11:59:21 +00:00
yukkop b12c35f957 fix: db-tool 2026-06-10 11:40:35 +00:00
yukkop bcf1b84dc4 fix: db-tool 2026-06-10 11:32:37 +00:00
yukkop 7c25e3b46d feat: db-tool: +secrets load 2026-06-09 23:57:51 +00:00
yukkop a20381e343 chore: ssh key 2026-06-09 22:35:06 +00:00
yukkop bd92610a98 feat: floating ip 2026-06-09 15:43:23 +00:00
yukkop e3ee881db6 chore: lab: neuro's ssh key 2026-06-09 15:11:41 +00:00
yukkop b9eabca464 feat: load-sops: +generic 2026-06-09 14:51:11 +00:00
yukkop fcc72192f5 dev: gitea: runners devshell 2026-06-08 10:22:34 +00:00
yukkop 5a0696ce64 ci: gitea: runners infra
runner nix smoke / nix label smoke (push) Has been cancelled
runner ubuntu smoke / ubuntu-latest label smoke (push) Has been cancelled
2026-06-08 08:18:08 +00:00
yukkop f4a59ff117 fix: mechabellum: 413 2026-06-07 23:14:14 +00:00
yukkop 129c82c863 chore: update mechabellum 2026-06-07 22:38:54 +00:00
yukkop 968c654320 fix: gitea: timeouts per iphone 429 errors 2026-06-07 22:24:11 +00:00
yukkop 98bb6c568f chore(package): gitea: verify heatmap package build 2026-06-06 23:01:57 +00:00
yukkop 72168aa8fa test(package): gitea: verify heatmap privacy endpoints
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-06 22:26:43 +00:00
yukkop 28dde5b9b1 test(package): gitea: cover heatmap author-date semantics
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus-Junior (openai/gpt-5.5) <clio-agent@sisyphuslabs.ai>
2026-06-06 22:10:18 +00:00
yukkop c2e0ba200c feat(package): gitea: wire heatmap reindexing
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-06 22:01:59 +00:00
yukkop 2eb23ea7ea feat(package): gitea: query heatmap by commit author date 2026-06-06 20:22:12 +00:00
yukkop 9906f71c5d feat(package): gitea: add private heatmap opt-in 2026-06-06 20:06:21 +00:00
yukkop 593c0d9abc feat(package): gitea: index heatmap commits by author date 2026-06-06 19:44:53 +00:00
yukkop 53dfd60b4c feat(package): gitea: add heatmap contribution model 2026-06-06 19:20:37 +00:00
111 changed files with 7990 additions and 512 deletions
+29
View File
@@ -0,0 +1,29 @@
name: runner nix smoke
on:
workflow_dispatch:
push:
branches:
- master
paths:
- .gitea/workflows/runner-nix-smoke.yaml
- flake.lock
- flake.nix
- infra/gitea-runners/**
jobs:
smoke:
name: nix label smoke
runs-on: nix
steps:
- name: Nix version and cache configuration
run: |
set -eu
nix --version
nix config show substituters
nix config show trusted-public-keys
- name: Repository flake evaluation
run: |
set -eu
nix flake check --no-build
+31
View File
@@ -0,0 +1,31 @@
name: runner ubuntu smoke
on:
workflow_dispatch:
push:
branches:
- master
paths:
- .gitea/workflows/runner-ubuntu-smoke.yaml
- infra/gitea-runners/**
jobs:
smoke:
name: ubuntu-latest label smoke
runs-on: ubuntu-latest
steps:
- name: Basic runner information
run: |
set -eu
echo "hello from gitea runner"
uname -a
- name: Docker smoke when available
run: |
set -eu
if command -v docker >/dev/null 2>&1; then
docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
docker run --rm hello-world
else
echo "docker command not available; skipping Docker smoke"
fi
+9
View File
@@ -47,6 +47,15 @@ creation_rules:
- *hectic-lab-server
- *umbriel-bfs
- path_regex: sus/gitea-runners.yaml$
key_groups:
- age:
- *nrv
- *yukkop
- *yukkop-alt
- *hectic-lab-server
- *umbriel-bfs
- path_regex: sus/matrix-cluster.yaml$
key_groups:
- age:
+21
View File
@@ -0,0 +1,21 @@
{
flake,
self,
inputs,
system ? "aarch64-darwin",
...
}: let
name = builtins.baseNameOf ./.;
in inputs.nix-darwin.lib.darwinSystem {
inherit system;
specialArgs = { inherit flake self inputs; };
modules = [
inputs.home-manager.darwinModules.home-manager
{
networking.hostName = name;
nixpkgs.hostPlatform = system;
nixpkgs.overlays = [ self.overlays.default ];
}
./${name}.nix
];
}
+177
View File
@@ -0,0 +1,177 @@
{
flake,
pkgs,
lib,
...
}: let
name = "yukkop";
in {
system.primaryUser = name;
nix.settings.experimental-features = "nix-command flakes";
programs.zsh.enable = true;
services.openssh.enable = true;
users.users.${name} = {
home = "/Users/${name}";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
];
};
environment.systemPackages = with pkgs; [
aerospace
git
moreutils
neovim
tmux
];
launchd.user.agents.aerospace = {
serviceConfig = {
ProgramArguments = [
"${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
];
RunAtLoad = true;
KeepAlive = true;
StandardOutPath = "/tmp/aerospace.out.log";
StandardErrorPath = "/tmp/aerospace.err.log";
};
};
home-manager.useGlobalPkgs = true;
home-manager.useUserPackages = true;
home-manager.backupFileExtension = "backup";
home-manager.sharedModules = [
(flake + "/home/module/program/tmux.nix")
];
home-manager.users.${name} = {
home.stateVersion = "25.11";
home.packages = with pkgs; [
iproute2mac
jujutsu
ripgrep
];
programs.git = {
enable = true;
lfs.enable = true;
settings = {
user.name = name;
user.email = "hectic.yukkop@gmail.com";
push.autoSetupRemote = true;
init.defaultBranch = "master";
};
};
programs.zsh = {
enable = true;
enableCompletion = true;
autosuggestion.enable = true;
syntaxHighlighting.enable = true;
history = {
size = 10000;
path = "$HOME/.zsh/.zsh_history";
};
shellAliases = {
drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
nv = "nvim";
tmux = "tmux a";
};
initContent = ''
export PATH=/Users/yukkop/.opencode/bin:$PATH
'';
};
xdg.configFile."aerospace/aerospace.toml".text = ''
start-at-login = false
enable-normalization-flatten-containers = true
enable-normalization-opposite-orientation-for-nested-containers = true
default-root-container-layout = 'tiles'
default-root-container-orientation = 'auto'
accordion-padding = 30
on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
automatically-unhide-macos-hidden-apps = false
[exec]
inherit-env-vars = true
[exec.env-vars]
PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
[gaps]
inner.horizontal = 8
inner.vertical = 8
outer.left = 8
outer.bottom = 8
outer.top = 8
outer.right = 8
[mode.main.binding]
alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
alt-slash = 'layout tiles horizontal vertical'
alt-comma = 'layout accordion horizontal vertical'
alt-f = 'fullscreen'
alt-h = 'focus left'
alt-j = 'focus down'
alt-k = 'focus up'
alt-l = 'focus right'
alt-shift-h = 'move left'
alt-shift-j = 'move down'
alt-shift-k = 'move up'
alt-shift-l = 'move right'
alt-minus = 'resize smart -50'
alt-equal = 'resize smart +50'
alt-1 = 'workspace 1'
alt-2 = 'workspace 2'
alt-3 = 'workspace 3'
alt-4 = 'workspace 4'
alt-5 = 'workspace 5'
alt-6 = 'workspace 6'
alt-7 = 'workspace 7'
alt-8 = 'workspace 8'
alt-9 = 'workspace 9'
alt-shift-1 = 'move-node-to-workspace 1'
alt-shift-2 = 'move-node-to-workspace 2'
alt-shift-3 = 'move-node-to-workspace 3'
alt-shift-4 = 'move-node-to-workspace 4'
alt-shift-5 = 'move-node-to-workspace 5'
alt-shift-6 = 'move-node-to-workspace 6'
alt-shift-7 = 'move-node-to-workspace 7'
alt-shift-8 = 'move-node-to-workspace 8'
alt-shift-9 = 'move-node-to-workspace 9'
alt-tab = 'workspace-back-and-forth'
alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
alt-shift-semicolon = 'mode service'
[mode.service.binding]
esc = ['reload-config', 'mode main']
r = ['flatten-workspace-tree', 'mode main']
f = ['layout floating tiling', 'mode main']
b = ['balance-sizes', 'mode main']
backspace = ['close-all-windows-but-current', 'mode main']
alt-shift-h = ['join-with left', 'mode main']
alt-shift-j = ['join-with down', 'mode main']
alt-shift-k = ['join-with up', 'mode main']
alt-shift-l = ['join-with right', 'mode main']
'';
};
system.stateVersion = 6;
}
+1
View File
@@ -8,6 +8,7 @@
haskell = import ./haskell.nix { inherit self system pkgs; };
neuro = import ./neuro.nix { inherit self system pkgs; };
xmpp = import ./xmpp.nix { inherit self system pkgs; };
gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
default = pkgs.mkShell {
buildInputs =
(with self.packages.${system}; [
+122
View File
@@ -0,0 +1,122 @@
{ pkgs, ... }: let
opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
tofu = pkgs.writeShellScriptBin "tofu" ''
exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
'';
giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
cat <<'EOF'
Gitea runners setup checklist
Tools available in this shell:
tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
Environment expected before real deploy/apply:
TF_VAR_hcloud_token
TF_VAR_ssh_public_key
TF_VAR_ssh_private_key
S3 backend credentials and endpoint access
a matching SOPS age identity for sus/gitea-runners.yaml
kubectl access to the target cluster
a concrete registry digest for the pushed Nix-capable runner image if enabling
the nix label
OpenTofu validation gate:
tofu version
tofu -chdir=infra/gitea-runners/opentofu validate
Nix image build/publish/digest gate:
nix build .#gitea-runner-nix-image
publish the archive, then pin the registry-reported digest in the runner label
mapping
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
SOPS token Secret creation gate:
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
umask 077
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
trap 'rm -f "$token_file"' EXIT
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
kubectl -n gitea-runners create secret generic gitea-runner-token \
--from-file=token="$token_file" \
--dry-run=client \
-o yaml | kubectl -n gitea-runners apply -f -
Cluster provision gate:
tofu -chdir=infra/gitea-runners/opentofu init
tofu -chdir=infra/gitea-runners/opentofu validate
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
Kubernetes apply gate:
kubectl config current-context
kubectl get nodes -o wide
kubectl get sc
kubectl apply -k infra/gitea-runners/k8s
Verification commands:
kubectl -n gitea-runners get statefulset gitea-runner
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
Main blockers and gates:
do not run tofu apply without all external inputs
do not apply the k8s overlay until the gitea-runner-token Secret exists
do not enable the nix label until the image has been published with a concrete digest
do not print, load, or require secrets on shell entry
EOF
'';
in pkgs.mkShell {
name = "gitea-runners";
buildInputs = [
tofu
giteaRunnersSetup
pkgs.nix
pkgs.kubectl
pkgs.kustomize
pkgs.kubeconform
pkgs.sops
pkgs.age
pkgs.awscli2
pkgs.hcloud
pkgs.tea
pkgs.docker
pkgs.skopeo
pkgs.go-containerregistry
pkgs.jq
pkgs.yq-go
pkgs.curl
pkgs.git
pkgs.openssh
];
shellHook = ''
export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
export GITEA_RUNNERS_NAMESPACE="gitea-runners"
alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
echo ""
echo "=== Gitea runner setup DevShell ==="
echo ""
echo "Run gitea-runners-setup for the full setup checklist."
echo "Paths: "
echo " root=$GITEA_RUNNERS_ROOT"
echo " tofu=$GITEA_RUNNERS_TOFU_DIR"
echo " k8s=$GITEA_RUNNERS_K8S_DIR"
echo " image=$GITEA_RUNNERS_IMAGE_DIR"
echo ""
'';
}
Generated
+26 -4
View File
@@ -675,11 +675,11 @@
]
},
"locked": {
"lastModified": 1780435429,
"narHash": "sha256-J69BMplOQOalw/RfY7NP7x4kd1Pozm5h4Ie8PpCc4gg=",
"lastModified": 1780905541,
"narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
"ref": "refs/heads/master",
"rev": "4f03d4f0ddc8483a180190828f6c756e428f6011",
"revCount": 113,
"rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
"revCount": 123,
"type": "git",
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
},
@@ -688,6 +688,27 @@
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
}
},
"nix-darwin": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1772129556,
"narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
"owner": "nix-darwin",
"repo": "nix-darwin",
"rev": "ebec37af18215214173c98cf6356d0aca24a2585",
"type": "github"
},
"original": {
"owner": "nix-darwin",
"ref": "nix-darwin-25.11",
"repo": "nix-darwin",
"type": "github"
}
},
"nix-minecraft": {
"inputs": {
"flake-compat": "flake-compat_3",
@@ -967,6 +988,7 @@
"hyprland": "hyprland",
"impermanence": "impermanence",
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
"nix-darwin": "nix-darwin",
"nix-minecraft": "nix-minecraft",
"nixos-anywhere": "nixos-anywhere",
"nixos-hardware": "nixos-hardware",
+10 -2
View File
@@ -42,6 +42,10 @@
url = "github:nix-community/home-manager/release-25.11";
inputs.nixpkgs.follows = "nixpkgs";
};
nix-darwin = {
url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-wsl = {
url = "github:nix-community/NixOS-WSL";
inputs.nixpkgs.follows = "nixpkgs";
@@ -119,8 +123,12 @@
# FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
"tenix|x86_64-linux" = import ./nixos/system/tenix { inherit flake self inputs; system = "x86_64-linux"; };
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
};
darwinConfigurations = {
"yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
};
};
}
+30
View File
@@ -0,0 +1,30 @@
{ pkgs, ... }: {
programs.tmux = {
enable = true;
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
keyMode = "vi";
escapeTime = 500;
historyLimit = 50000;
newSession = true;
extraConfig = ''
# resurrect
set -g @resurrect-strategy-vim 'session'
set -g @resurrect-strategy-nvim 'session'
set -g @resurrect-capture-pane-contents 'on'
resurrect_dir="$HOME/.tmux/resurrect"
set -g @resurrect-dir $resurrect_dir
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
# continuum
set -g @continuum-restore 'on'
set -g @continuum-boot 'on'
set -g @continuum-save-interval '10'
bind-key -T copy-mode-vi v send-keys -X begin-selection
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
bind-key O select-pane -t :.-
'';
};
}
+80
View File
@@ -0,0 +1,80 @@
# Gitea runner Nix image
The repo-owned Nix-capable job image is built by the flake package
`gitea-runner-nix-image`.
```sh
nix build .#gitea-runner-nix-image
```
The package emits a Docker archive with the local build tag:
```text
gitea-runner-nix-image:2026-06-07
```
That tag is build metadata only. Do not use it as the final Gitea runner label
mapping because runner job images must be immutable.
## Publication target
Preferred registry:
```text
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
```
Publish the archive without adding secrets to the image layers, then use the
registry-reported digest as the only final `nix` label image reference:
```text
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
```
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
runner label mapping must use the `@sha256:` reference above. Keep
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
configuration task explicitly changes it. Only the `nix` label should select
this custom image.
If the Gitea container registry is unavailable, select a private registry that
is reachable from the runner Kubernetes cluster and requires authentication that
can be provided through Kubernetes image-pull secrets. Record the selected
registry and replace the host in the same digest-pinned form:
```text
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
```
Do not fall back to `latest` or a tag-only mapping.
## Task 7 publication status
Local build evidence is recorded in
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
load and tag the image, but pushing to the preferred registry failed with
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
concrete final mapping. Kubernetes pull smoke is recorded in
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
is not installed or not on `PATH`.
Once registry credentials are available, rerun the push, capture the
registry-reported digest, and replace `<registry-digest>` in the mapping above
before Task 6/9 consumes the label configuration.
## Image contents
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
top-level `flake.nix`:
```text
experimental-features = nix-command flakes
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
sandbox = false
```
No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
credential belongs in this image. Runtime secrets stay with the Kubernetes
runner configuration and token-file mount contract.
@@ -0,0 +1,154 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
rules:
- apiGroups:
- ""
resources:
- pods
- persistentvolumeclaims
verbs:
- get
- list
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
subjects:
- kind: ServiceAccount
name: gitea-runner-lifecycle
namespace: gitea-runners
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: gitea-runner-lifecycle
---
apiVersion: v1
kind: ConfigMap
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
data:
cleanup-dry-run.sh: |
#!/bin/sh
set -eu
namespace="${RUNNER_NAMESPACE:-gitea-runners}"
mode="${CLEANUP_MODE:-dry-run}"
selector="app.kubernetes.io/name=gitea-runner"
if [ "$namespace" != "gitea-runners" ]; then
printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
exit 13
fi
if [ "$mode" != "dry-run" ]; then
printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
exit 13
fi
printf 'gitea runner lifecycle cleanup dry-run\n'
printf 'namespace: %s\n' "$namespace"
printf 'mode: %s\n\n' "$mode"
printf 'StatefulSet:\n'
kubectl -n "$namespace" get statefulset gitea-runner -o wide
printf '\nActive runner pods:\n'
kubectl -n "$namespace" get pods -l "$selector" -o wide
printf '\nRunner /data PVCs:\n'
kubectl -n "$namespace" get pvc -l "$selector" -o wide
printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
found_candidate=0
for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
pod="${pvc#data-}"
if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
found_candidate=1
printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
fi
done
if [ "$found_candidate" -eq 0 ]; then
printf 'none\n'
fi
printf '\nGitea registration reconciliation:\n'
printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: gitea-runner-cleanup-dry-run
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
spec:
schedule: "17 3 * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
ttlSecondsAfterFinished: 3600
template:
metadata:
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
spec:
serviceAccountName: gitea-runner-lifecycle
restartPolicy: Never
containers:
- name: cleanup-dry-run
image: bitnami/kubectl:1.30
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- /scripts/cleanup-dry-run.sh
env:
- name: RUNNER_NAMESPACE
value: gitea-runners
- name: CLEANUP_MODE
value: dry-run
volumeMounts:
- name: lifecycle-scripts
mountPath: /scripts
readOnly: true
volumes:
- name: lifecycle-scripts
configMap:
name: gitea-runner-lifecycle
defaultMode: 0555
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- service.yaml
- service-account.yaml
- runner-config.yaml
- statefulset.yaml
- cleanup-lifecycle.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
@@ -0,0 +1,36 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: gitea-runner-config
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
data:
config.yaml: |
log:
level: info
runner:
file: /data/.runner
capacity: 1
envs: {}
timeout: 3h
insecure: false
fetch_timeout: 5s
fetch_interval: 2s
labels:
- ubuntu-latest
# The nix label is intentionally disabled until the runner image has a
# concrete registry-reported digest; see ../runbook.md before deploy.
cache:
enabled: true
dir: /data/cache
container:
network: bridge
privileged: false
force_pull: true
valid_volumes: []
docker_host: unix:///runner-docker/docker.sock
@@ -0,0 +1,36 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
rules: []
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
subjects:
- kind: ServiceAccount
name: gitea-runner
namespace: gitea-runners
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: gitea-runner
+16
View File
@@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
spec:
clusterIP: None
selector:
app.kubernetes.io/name: gitea-runner
ports:
- name: cache
port: 8088
targetPort: cache
+156
View File
@@ -0,0 +1,156 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
spec:
serviceName: gitea-runner
replicas: 5
podManagementPolicy: Parallel
selector:
matchLabels:
app.kubernetes.io/name: gitea-runner
template:
metadata:
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
annotations:
hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
spec:
serviceAccountName: gitea-runner
automountServiceAccountToken: false
terminationGracePeriodSeconds: 60
securityContext:
fsGroup: 1000
containers:
- name: runner
image: gitea/act_runner:0.2.11
imagePullPolicy: IfNotPresent
env:
- name: GITEA_INSTANCE_URL
value: https://gitea.hectic-lab.com
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
value: /runner-secrets/token
- name: CONFIG_FILE
value: /runner-config/config.yaml
- name: DOCKER_HOST
value: unix:///runner-docker/docker.sock
ports:
- name: cache
containerPort: 8088
resources:
requests:
cpu: 250m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
exec:
command:
- /bin/sh
- -ec
- test -s /data/.runner && test -S /runner-docker/docker.sock
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
exec:
command:
- /bin/sh
- -ec
- test -S /runner-docker/docker.sock
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
volumeMounts:
- name: data
mountPath: /data
- name: config
mountPath: /runner-config
readOnly: true
- name: runner-token
mountPath: /runner-secrets
readOnly: true
- name: docker-socket
mountPath: /runner-docker
- name: docker
image: docker:27-dind
imagePullPolicy: IfNotPresent
args:
- --host=unix:///runner-docker/docker.sock
- --storage-driver=overlay2
- --tls=false
env:
- name: DOCKER_TLS_CERTDIR
value: ""
- name: DOCKER_HOST
value: unix:///runner-docker/docker.sock
securityContext:
# Privileged rootful DinD is intentionally scoped to this trusted
# internal runner pool; never expose it to untrusted fork/PR jobs.
privileged: true
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
exec:
command:
- docker
- info
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 6
readinessProbe:
exec:
command:
- docker
- info
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 6
volumeMounts:
- name: docker-socket
mountPath: /runner-docker
- name: docker-graph
mountPath: /var/lib/docker
volumes:
- name: config
configMap:
name: gitea-runner-config
- name: runner-token
secret:
secretName: gitea-runner-token
items:
- key: token
path: token
defaultMode: 0400
- name: docker-socket
emptyDir: {}
- name: docker-graph
emptyDir: {}
volumeClaimTemplates:
- metadata:
name: data
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes
resources:
requests:
storage: 20Gi
+29
View File
@@ -0,0 +1,29 @@
# OpenTofu working directory and downloaded modules/providers.
.terraform/
.terraform.lock.hcl
# State must live in the S3 backend for production. Local state is allowed only
# for throwaway syntax checks with `tofu init -backend=false` and must not be
# committed.
terraform.tfstate
terraform.tfstate.*
*.tfstate
*.tfstate.*
crash.log
crash.*.log
# Plans can contain secrets or derived infrastructure data.
*.tfplan
*.plan
kubeconfig
kubeconfig.yaml
*_kubeconfig.yaml
# Variable files commonly carry credentials. Keep production inputs in SOPS or
# external environment/configuration, not in checked-in files.
*.tfvars
*.tfvars.json
override.tf
override.tf.json
*_override.tf
*_override.tf.json
+90
View File
@@ -0,0 +1,90 @@
# Gitea runner OpenTofu backend contract
This directory defines the safe backend, provider contract, and kube-hetzner
cluster stack for the Gitea runner Kubernetes cluster.
## Required backend
Production state must use the OpenTofu S3 backend in `backend.tf`:
- bucket: `gitea-runner-hectic-lab`
- key: `gitea-runners/kube-hetzner/terraform.tfstate`
- region: `fsn1`, aligned with the target Hetzner location
- encryption: `encrypt = true`
- locking: `use_lockfile = true` where the selected S3-compatible endpoint
supports it
Before any production `tofu init`, verify the S3-compatible endpoint, credential
source, bucket versioning, encryption behavior, and lockfile support for the
chosen object-storage provider. Keep backend authentication externalized through
environment variables, AWS-compatible shared config, or the production secret
injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
OpenTofu files.
## Local state safety
Production local state is forbidden. Only syntax-only validation/prototyping may
use local state, and it must use backend-disabled initialization:
```sh
tofu -chdir=infra/gitea-runners/opentofu init -backend=false
tofu -chdir=infra/gitea-runners/opentofu validate
```
Fail the run if production local state appears:
```sh
test ! -e infra/gitea-runners/opentofu/terraform.tfstate
test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
grep -R 'backend "s3"' infra/gitea-runners/opentofu
```
The `.gitignore` in this directory blocks local state, plans, downloaded
providers/modules, and variable files from being committed. Treat any local
state file as disposable validation residue, never as production state.
## Provider and module pins
`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
as locals so Task 5 can wire the module without re-opening the version contract.
`providers.tf` leaves the `hcloud` provider empty so authentication comes from
the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
Do not set token values in `.tf` or `.tfvars` files.
## Cluster shape
The default cluster is deliberately fixed-size:
- cluster name: `gitea-runners`
- Hetzner location: `fsn1`
- private network region: `eu-central`
- control plane: one `cpx21` node in pool `control-plane`
- workers: three `cpx31` nodes in pool `runner-workers`
- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
- Longhorn: disabled
- autoscaling/KEDA: not enabled in this stack
The three default workers are sized for the initial five trusted privileged DinD
jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
`worker_count` to `5` or increase `worker_server_type`, then run a fresh
`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
Required inputs must come from environment or secret injection, for example
`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
enabled; this path is ignored as operational secret material.
## Known state caveat
kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
internal `kube_system_secrets` handling. This task does not claim that risk is
solved. Task 5 must verify the generated plan and state before production apply
and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
keys, and decrypted secrets are absent from committed files and unsafe state
evidence.
+16
View File
@@ -0,0 +1,16 @@
terraform {
backend "s3" {
bucket = "gitea-runner-hectic-lab"
key = "gitea-runners/kube-hetzner/terraform.tfstate"
region = "fsn1"
encrypt = true
use_lockfile = true
}
}
check "remote_state_contract" {
assert {
condition = local.production_remote_state
error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
}
}
+60
View File
@@ -0,0 +1,60 @@
locals {
default_storage_class = "hcloud-volumes"
control_plane_nodepools = [
{
name = "control-plane"
server_type = var.control_plane_server_type
location = var.hetzner_location
labels = []
taints = []
count = 1
},
]
agent_nodepools = [
{
name = "runner-workers"
server_type = var.worker_server_type
location = var.hetzner_location
labels = ["node-role.hectic-lab/gitea-runner=true"]
taints = []
count = var.worker_count
},
]
}
module "kube_hetzner" {
source = "kube-hetzner/kube-hetzner/hcloud"
version = "2.19.3"
providers = {
hcloud = hcloud
}
hcloud_token = var.hcloud_token
ssh_public_key = var.ssh_public_key
ssh_private_key = var.ssh_private_key
cluster_name = var.cluster_name
base_domain = var.base_domain
# kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
# expose that expected path without outputting kubeconfig private key material.
create_kubeconfig = true
network_region = var.network_region
load_balancer_location = var.hetzner_location
control_plane_nodepools = local.control_plane_nodepools
agent_nodepools = local.agent_nodepools
# Hetzner CSI is the required StorageClass provider for runner PVCs.
disable_hetzner_csi = false
# Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
enable_longhorn = false
# Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
# raise worker_count to 5 or increase worker_server_type after validating pod
# CPU, memory, and ephemeral-storage pressure in Task 11.
}
+22
View File
@@ -0,0 +1,22 @@
output "kubeconfig_path" {
description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
}
output "cluster_name" {
description = "kube-hetzner cluster name."
value = var.cluster_name
}
output "node_pool_names" {
description = "Control-plane and worker node pool names used by this stack."
value = {
control_plane = [for pool in local.control_plane_nodepools : pool.name]
workers = [for pool in local.agent_nodepools : pool.name]
}
}
output "default_storage_class" {
description = "Default Hetzner CSI StorageClass expected for runner PVCs."
value = local.default_storage_class
}
@@ -0,0 +1 @@
provider "hcloud" {}
+74
View File
@@ -0,0 +1,74 @@
variable "hcloud_token" {
description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
type = string
sensitive = true
}
variable "ssh_public_key" {
description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
type = string
}
variable "ssh_private_key" {
description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
type = string
sensitive = true
}
variable "cluster_name" {
description = "Name for the kube-hetzner runner cluster."
type = string
default = "gitea-runners"
validation {
condition = can(regex("^[a-z0-9-]+$", var.cluster_name))
error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
}
}
variable "hetzner_location" {
description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
type = string
default = "fsn1"
}
variable "network_region" {
description = "Hetzner private network region. eu-central covers fsn1."
type = string
default = "eu-central"
}
variable "control_plane_server_type" {
description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
type = string
default = "cpx21"
}
variable "worker_server_type" {
description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
type = string
default = "cpx31"
}
variable "worker_count" {
description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
type = number
default = 3
validation {
condition = var.worker_count >= 1
error_message = "worker_count must be at least 1."
}
}
variable "kubeconfig_path" {
description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
type = string
default = null
}
variable "base_domain" {
description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
type = string
default = ""
}
+17
View File
@@ -0,0 +1,17 @@
terraform {
required_version = ">= 1.10.1"
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = "1.60.1"
}
}
}
locals {
kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud"
kube_hetzner_module_version = "2.19.3"
hcloud_provider_minimum = ">= 1.59.0"
production_remote_state = true
}
+503
View File
@@ -0,0 +1,503 @@
# Gitea Runner Infrastructure Runbook
## Scope
This directory is the repo-owned boundary for the first Gitea Actions runner
pool. Task 1 only establishes the scaffold and immutable decision contract;
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
and a Nix-capable runner image under the existing subdirectories.
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
workflow job containers are ephemeral, while each runner pod keeps its runner
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
## Immutable decisions
- Infrastructure is managed with OpenTofu command examples only, using the
`tofu` CLI.
- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
- Runner implementation is the non-Enterprise `gitea/runner`.
- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
environment variables are not the contract.
- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
`/data`, including `/data/.runner`.
- Container builds run through privileged rootful DinD inside trusted runner
pods; host Docker socket mounting is not an implementation path.
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
the Nix-capable image has been pushed and a concrete registry-reported digest
is added to the runner ConfigMap.
- First scope is trusted internal workflows only, with no untrusted fork or PR
workflow support.
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
## Lifecycle boundaries
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
contract, Hetzner provider configuration, and kube-hetzner module wiring.
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
handoff; package or flake output changes are outside Task 1.
- `infra/gitea-runners/runbook.md`: this contract plus later operational
commands, rollback notes, and acceptance evidence references.
## Guardrails
- Enterprise ARC/actions-runner-controller are rejected alternatives and must
not be implemented here. Do not add ARC custom resources, controller install
instructions, or GitHub Actions ARC assumptions.
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
acceptable for trusted internal jobs.
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
runner pool.
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
S3 credentials, decrypted SOPS files, or SOPS age keys.
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
runner token; Kubernetes receives the token as a mounted file secret instead.
- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
jobs must reach the public HTTPS service.
## Initial acceptance commands
Run from the repository root:
```sh
test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
test -f infra/gitea-runners/runbook.md
grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
grep -R "[t]erraform " infra/gitea-runners || true
grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
```
Expected outcomes: the directory and file checks exit 0; the architecture-term
grep shows this contract; ARC references appear only in the rejected-alternative
guardrail above; there are no forbidden CLI command examples and no unresolved
decision placeholders.
## Downstream placeholders
- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
- Task 3: add SOPS secret contract and runtime token delivery details.
- Task 4: define or package the Nix-capable runner image for the `nix` label.
- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
and document cleanup, rollback, and scaling operations.
## Runner lifecycle cleanup
All lifecycle commands are scoped to the runner namespace:
```sh
kubectl -n gitea-runners get statefulset gitea-runner
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
```
The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
does not delete pods, PVCs, Docker data, or Gitea runner registrations.
Run the same inventory on demand without waiting for the schedule:
```sh
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
```
The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
operator to remove finished jobs manually.
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
pod loses `/data/.runner`. That runner identity must then be deregistered from
Gitea or the replacement pod must be allowed to re-register intentionally with
the current organization runner token. Do not delete an active runner PVC as a
normal cleanup step.
Non-UI Gitea registration reconciliation uses the Gitea API with a separate
admin token. Store that token outside this repository and pass it as a file; do
not print it:
```sh
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
--restart=Never \
--image=curlimages/curl:8.10.1 \
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
```
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
has completed and its logs have been collected. Do not keep this admin token in
the runner namespace longer than the reconciliation window.
Only remove a stale Gitea runner registration after the corresponding pod/PVC
was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
cleanup is a fallback, not the only path. Record the removed runner name and the
Kubernetes PVC/pod deletion that made it stale.
After the dry-run list identifies a stale registration and the PVC/pod deletion
has been recorded, remove that exact Gitea runner by id through the API:
```sh
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
umask 077
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
trap 'rm -f "$curl_config"' EXIT
{
printf 'request = "DELETE"\n'
printf 'header = "Authorization: token '
cat /secure/path/gitea-admin-token
printf '"\n'
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
} > "$curl_config"
curl -fsS --config "$curl_config"
```
Do not run the delete command for a runner that still has an active
`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
being intentionally reset for re-registration.
## Docker-in-Docker storage cleanup
Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
list disk usage before pruning, and run the command only against the `docker`
container in runner pods in `gitea-runners`. Because this storage is pod-local,
loop over pods for pool-wide cleanup:
```sh
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
```
For one pod, replace the StatefulSet target with the pod name:
```sh
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
```
Do not run host-level Docker cleanup commands and do not mount or prune a host
Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
Kubernetes; the `/data` PVC remains and still controls runner identity.
## Token rotation
Rotate the Gitea organization runner token without printing decrypted values:
```sh
sops sus/gitea-runners.yaml
umask 077
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
trap 'rm -f "$token_file"' EXIT
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
kubectl -n gitea-runners create secret generic gitea-runner-token \
--from-file=token="$token_file" \
--dry-run=client \
-o yaml | kubectl -n gitea-runners apply -f -
kubectl -n gitea-runners rollout restart statefulset/gitea-runner
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
```
The `rollout restart` command above is the controlled restart path for this
StatefulSet. Observe the rollout and each ordinal until all replacement pods are
Ready; do not delete runner pods directly as part of normal token rotation:
```sh
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
```
Verification must confirm the token file mount remains present while the token
value never appears in logs or evidence:
```sh
kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
```
## Deploy and status
These commands are executable only when the external inputs are available:
- `TF_VAR_hcloud_token`
- `TF_VAR_ssh_public_key`
- `TF_VAR_ssh_private_key`
- S3 backend credentials and endpoint access
- a matching SOPS age identity for `sus/gitea-runners.yaml`
- `kubectl` access to the target cluster
- a concrete digest for the pushed Nix-capable runner image, if enabling the
`nix` label
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
stale kubeconfig files.
Before production Kubernetes apply or rollout, satisfy both manifest gates:
1. Create or update the `gitea-runner-token` Secret from SOPS. The active
Kustomize overlay intentionally does not include a placeholder Secret, but
the StatefulSet still mounts `secretName: gitea-runner-token` as
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
image has been pushed successfully. Enable the `nix` label only by adding a
digest-pinned `docker://` mapping with the exact registry-reported sha256
digest from that push.
Use the same SOPS materialization pattern as token rotation before applying the
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
target namespace; the full overlay remains gated on the Secret and digest
decisions:
```sh
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
umask 077
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
trap 'rm -f "$token_file"' EXIT
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
kubectl -n gitea-runners create secret generic gitea-runner-token \
--from-file=token="$token_file" \
--dry-run=client \
-o yaml | kubectl -n gitea-runners apply -f -
```
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
above succeeds. Do not claim or enable the `nix` runner label until the image
publication step has produced the concrete digest.
```sh
tofu -chdir=infra/gitea-runners/opentofu init
tofu -chdir=infra/gitea-runners/opentofu validate
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
kubectl config current-context
kubectl get nodes -o wide
kubectl get sc
kubectl apply -k infra/gitea-runners/k8s
kubectl -n gitea-runners get statefulset gitea-runner
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
```
Expected status after deploy:
- `kubectl config current-context` names the runner cluster context.
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
## Scale 5 to 10 to 5
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
```sh
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
# Run the concurrent smoke workflows now.
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
```
After scaling back down, inspect the cleanup dry-run and deregister any stale
runner registrations only for pods or PVCs that were intentionally removed.
## Cleanup and stale runner deregistration
Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
PVC candidates whose pod is gone. It must not delete active resources.
```sh
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
```
Pool-wide DinD storage checks and cleanup:
```sh
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
```
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
replacement pod re-register intentionally with the current organization token.
Never delete an active runner PVC as routine cleanup.
Non-UI Gitea registration reconciliation uses an admin token stored outside this
repository:
```sh
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
--restart=Never \
--image=curlimages/curl:8.10.1 \
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
```
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
pod has completed and its logs have been collected.
After the dry-run list identifies a stale registration and the PVC or pod
deletion has been recorded, remove that exact Gitea runner by id through the
API:
```sh
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
umask 077
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
trap 'rm -f "$curl_config"' EXIT
{
printf 'request = "DELETE"\n'
printf 'header = "Authorization: token '
cat /secure/path/gitea-admin-token
printf '"\n'
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
} > "$curl_config"
curl -fsS --config "$curl_config"
```
Do not run the delete command for a runner that still has an active
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
being intentionally reset for re-registration.
## Application rollback
Rollback the app layer only. Do not use this section to destroy the cluster.
```sh
kubectl -n gitea-runners rollout history statefulset/gitea-runner
kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
```
If a manifest rollback is needed, reapply the repo overlay after checking out the
known-good revision, then re-run the rollout checks:
```sh
kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
```
## Full cluster teardown
This destroys Hetzner resources owned by the kube-hetzner stack, including the
`gitea-runners` cluster nodes, the `control-plane` node pool, the
`runner-workers` node pool, the cluster network, load balancer resources,
firewall objects, and any attached Hetzner CSI volumes still managed by the
stack. Do not run teardown unless the destruction is intentional.
```sh
tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
```
## Partial OpenTofu apply recovery
If `tofu apply` fails after creating some resources, do not destroy blindly.
First reconcile state and inspect what the stack thinks exists:
```sh
tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
tofu -chdir=infra/gitea-runners/opentofu state list
```
Then rerun the normal plan path. Use `-target` only as a last resort when a
single resource is stuck and the drift is understood.
## S3 backend recovery
If backend init or state access fails, first verify the bucket and versioning
outside OpenTofu, then reconfigure the backend:
```sh
nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
tofu -chdir=infra/gitea-runners/opentofu plan
```
If the backend reports a stale lock, confirm no `tofu` process is active, then
use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
unlock a live plan or apply.
## Gitea outage troubleshooting
Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
containers.
```sh
kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
```
If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
`/data/.runner` just because the service is unavailable. Once Gitea returns,
repeat the token rotation or re-registration path if a pod restarted while the
service was unavailable and lost its runner identity.
## Release checklist
Do not release unless the following evidence files exist and are readable:
- `.sisyphus/evidence/task-5-cluster-plan.txt`
- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
- `.sisyphus/evidence/task-9-deploy.txt`
- `.sisyphus/evidence/task-9-secret-mount.txt`
- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
- `.sisyphus/evidence/task-10-nix-workflow.txt`
- `.sisyphus/evidence/task-11-scale.txt`
- `.sisyphus/evidence/task-11-restart-cleanup.txt`
If any evidence file is missing, stop and collect it before treating the runbook
as complete.
+45 -9
View File
@@ -26,12 +26,39 @@
"aarch64-darwin"
];
cudaUnfreeNames = [
"cuda_nvcc"
"cuda_cudart"
"cuda_cuobjdump"
"cuda_cupti"
"cuda_nvdisasm"
"cuda_cccl"
"cuda_nvml_dev"
"cuda_nvrtc"
"cuda_nvtx"
"cuda_profiler_api"
"libcusparse_lt"
"libcublas"
"libcufft"
"libcufile"
"libcurand"
"libcusolver"
"libnvjitlink"
"libcusparse"
"cudnn"
];
cudaUnfreePredicate = pkg:
builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
builtins.foldl' (
acc: system: let
pkgs = import nixpkgs {
inherit system;
overlays = pkgOverlays;
config.allowUnfreePredicate = cudaUnfreePredicate;
};
systemOutputs = f {
system = system;
@@ -58,7 +85,7 @@
else {};
in {
# -- For all systems --
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems;
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
forSystems = systems: nixpkgs.lib.genAttrs systems;
forAllSystems = nixpkgs.lib.genAttrs AllSystems;
@@ -67,6 +94,7 @@ in {
logs = builtins.readFile ./shell/logs.sh;
check-tool = builtins.readFile ./shell/check-tool.sh;
local-dir = builtins.readFile ./shell/local-dir.sh;
load-sops = builtins.readFile ./shell/load-sops.sh;
};
sharedShellAliases = {
@@ -102,7 +130,7 @@ in {
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
# for everything that creates objects in the `hectic` namespace, used by
# migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply
# migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
# the full bundle via lib/hook/apply-hectic-bundle.sh.
#
# The whole hectic system shares one `versionString`; `hectic-version.sql`
@@ -187,19 +215,27 @@ in {
readModulesRecursive' = path: extraArgs:
with lib;
with builtins; let
paths = pipe "${path}" [
(filesystem.listFilesRecursive)
(filter (hasSuffix ".nix"))
];
collectPaths = dir: prefix:
concatLists (mapAttrsToList (name: type: let
path' = dir + "/${name}";
name' = if prefix == "" then name else "${prefix}/${name}";
in
if type == "directory"
then collectPaths path' name'
else [{
inherit path';
name = name';
}]
) (readDir dir));
paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
pathToName = flip pipe [
(removePrefix "${path}/")
(replaceStrings ["/" ".nix"] ["." ""])
(removeSuffix ".nix")
];
attrList =
map (path': {
name = pathToName (unsafeDiscardStringContext path');
value = import path' extraArgs;
name = pathToName path'.name;
value = import path'.path' extraArgs;
})
paths;
in
+4 -6
View File
@@ -10,8 +10,8 @@
# Usage:
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
#
# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via
# hectic.load_secrets_from_env() after the bundle is applied.
# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
# SQL file paths are substituted by Nix evaluation time.
apply_hectic_bundle() {
@@ -41,11 +41,9 @@ apply_hectic_bundle() {
done
if [ -n "$env_content" ]; then
# Dollar-quote with $ps_env$ tag to preserve all content verbatim.
env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
SELECT hectic.load_secrets_from_env(\$ps_env\$
$env_content
\$ps_env\$);
SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
SQL
fi
+6 -5
View File
@@ -4,8 +4,8 @@ Single source of truth for every object created in the `hectic` PostgreSQL
schema. Consumed by:
- `package/migrator` — applies the bundle on `migrator init` (mandatory).
- `package/db-tool` — applies the bundle in `database hydrate` (default; opt
out with `--no-hook`).
- `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
(default; opt out with `--no-hook`) and in `db-ops secrets load`.
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
paths exposed via `self.lib.hectic.*.path`.
@@ -60,7 +60,7 @@ verbatim source files in the store.
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
SQL paths embedded at Nix evaluation time. `migrator` and `db-tool` splice that
SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
shell source directly into their generated scripts. Public entry point:
```sh
@@ -87,8 +87,9 @@ External consumers that do not want to source the helper can still invoke
4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
update `lib/hook/apply-hectic-bundle.sh` to apply the file.
5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`
and `test/package/db-tool/test/postgresql/hydrate-hook/`.
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
`test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
bundle-loading coverage.
## Versioning
+204
View File
@@ -0,0 +1,204 @@
load_sops_shell_quote() {
printf "'"
printf '%s' "$1" | sed "s/'/'\"'\"'/g"
printf "'"
}
load_sops_normalize_key() {
load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
case "$load_sops_normalized_key" in
''|[!A-Z_]*|*[!A-Z0-9_]*)
printf 'load-sops: invalid environment name after key normalization\n' >&2
return 1
;;
esac
printf '%s\n' "$load_sops_normalized_key"
}
load_sops_env_from_sops_file() {
load_sops_file=$1
load_sops_extract=${2-}
if ! command -v sops >/dev/null 2>&1; then
printf 'load-sops: required tool `sops` not found\n' >&2
return 1
fi
if ! command -v yq >/dev/null 2>&1; then
printf 'load-sops: required tool `yq` not found\n' >&2
return 1
fi
load_sops_decrypted=''
load_sops_attempt=0
load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
while :; do
if [ -n "$load_sops_extract" ]; then
if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
load_sops_status=0
else
load_sops_status=$?
fi
else
if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
load_sops_status=0
else
load_sops_status=$?
fi
fi
if [ "$load_sops_status" -eq 0 ]; then
break
fi
if [ "$load_sops_prompt" != 1 ]; then
printf 'load-sops: failed to decrypt file\n' >&2
return 1
fi
if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
return 1
fi
load_sops_attempt=$((load_sops_attempt + 1))
if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
printf 'load-sops: decrypt failed after configured retries\n' >&2
return 1
fi
load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
case "$load_sops_use_script" in
auto)
if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
load_sops_script_status=0
else
load_sops_script_status=$?
fi
if [ "$load_sops_script_status" -eq 0 ]; then
continue
fi
fi
;;
1)
if ! command -v script >/dev/null 2>&1; then
printf 'load-sops: required tool `script` not found\n' >&2
return 1
fi
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
load_sops_script_status=0
else
load_sops_script_status=$?
fi
if [ "$load_sops_script_status" -eq 0 ]; then
continue
fi
;;
0)
;;
*)
printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
return 1
;;
esac
printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
printf 'load-sops: failed to read prompt input\n' >&2
return 1
fi
export SOPS_AGE_KEY_CMD
done
load_sops_env_from_yaml_text "$load_sops_decrypted"
}
load_sops_env_from_yaml_file() {
load_sops_file=$1
if ! command -v yq >/dev/null 2>&1; then
printf 'load-sops: required tool `yq` not found\n' >&2
return 1
fi
load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
}
load_sops_env_from_yaml_text() {
load_sops_yaml=$1
load_sops_seen=''
if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
load_sops_keys_status=0
else
load_sops_keys_status=$?
fi
if [ "$load_sops_keys_status" -ne 0 ]; then
printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
return 1
fi
while IFS= read -r load_sops_key; do
[ -n "$load_sops_key" ] || continue
load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
case "
$load_sops_seen
" in
*"
$load_sops_name
"*)
if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
printf 'load-sops: normalized environment name collision\n' >&2
return 1
fi
;;
esac
load_sops_seen=${load_sops_seen}${load_sops_seen:+"
"}$load_sops_name
load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
printf 'load-sops: failed to inspect YAML value kind\n' >&2
return 1
}
load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
printf 'load-sops: failed to inspect YAML value tag\n' >&2
return 1
}
if [ "$load_sops_kind" != scalar ]; then
printf 'load-sops: top-level YAML values must be scalars\n' >&2
return 1
fi
if [ "$load_sops_tag" = '!!null' ]; then
printf 'load-sops: top-level YAML null values are not supported\n' >&2
return 1
fi
if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
if [ -n "$load_sops_already_set" ]; then
continue
fi
fi
load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
printf 'load-sops: failed to read YAML scalar value\n' >&2
return 1
}
load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
eval "export $load_sops_name=$load_sops_quoted"
done <<EOF
$load_sops_keys
EOF
}
+2 -2
View File
@@ -9,11 +9,11 @@ with self.lib;
let
# Combine hectic modules into one
hectic.imports = attrValues (
readModulesRecursive' ./hectic { inherit flake self inputs; }
readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
);
# Read generic modules separately
generic = readModulesRecursive'
./generic
(flake + "/nixos/module/generic")
{ inherit flake self inputs; };
in generic // {
inherit hectic;
@@ -18,6 +18,7 @@
];
adminNames = [ "yukkop" ];
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
cfg = config.hectic.generic.matrix-cluster;
in {
@@ -39,7 +40,7 @@ in {
value = {
key = "matrix/users/${name}/password";
owner = "matrix-synapse";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
}) userNames
);
+7
View File
@@ -250,6 +250,7 @@ in {
# failover flip does not need a separate provisioning step.
systemd.tmpfiles.rules = [
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
"Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
];
systemd.services.matrix-cluster-signing-key = {
@@ -319,6 +320,12 @@ in {
media_store_path = s3Cfg.mediaStorePath;
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
# Tolerate bursty Element/iPhone presence syncs without disabling limits.
rc_presence.per_user = {
per_second = 0.5;
burst_count = 5;
};
experimental_features = {
msc3266_enabled = true;
msc4140_enabled = true;
+53 -11
View File
@@ -1,7 +1,5 @@
{
inputs,
flake,
self,
{
...
}:
{
pkgs,
@@ -11,6 +9,13 @@
}: let
cfg = config.hectic.hardware.hetzner-cloud;
isNewer = cfg.generation == "newer";
networkMatchConfig =
(lib.optionalAttrs (cfg.networkMatchConfigName != null) {
Name = cfg.networkMatchConfigName;
})
// (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
PermanentMACAddress = cfg.networkMatchConfigMac;
});
in {
options.hectic.hardware.hetzner-cloud = {
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
@@ -51,6 +56,14 @@ in {
'';
};
floatingIpv4 = lib.mkOption {
type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
default = null;
example = "188.243.124.247";
description = ''
Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
'';
};
device = lib.mkOption {
type = lib.types.str;
default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
@@ -65,14 +78,27 @@ in {
'';
};
networkMatchConfigName = lib.mkOption {
type = lib.types.strMatching "^(enp1s0|ens3|eth0)$";
type = with lib.types; nullOr str;
default = null;
example = "enp1s0";
description = ''
type of network conection,
on older hetzner servers may be `ens3` or else
on newer probably `enp1s0`
Optional interface name to match in systemd-networkd.
you can use `networkctl list` on server to know it
Prefer `networkMatchConfigMac` for stable matching across rescue
images and installed systems that may rename interfaces differently.
You can use `networkctl list` on server to know it.
'';
};
networkMatchConfigMac = lib.mkOption {
type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
default = null;
example = "92:00:08:4a:b0:32";
description = ''
Optional permanent MAC address to match in systemd-networkd.
This is the preferred Hetzner Cloud matching method because interface
names can differ between rescue images and installed NixOS systems.
'';
};
};
@@ -80,6 +106,15 @@ in {
config = lib.mkIf cfg.enable (lib.mkMerge
[
{
boot.loader.systemd-boot.enable = false;
boot.loader.efi.canTouchEfiVariables = false;
boot.loader.grub = {
enable = true;
efiSupport = true;
efiInstallAsRemovable = true;
device = "nodev";
};
boot.initrd.availableKernelModules = [
"ata_piix"
"uhci_hcd"
@@ -90,12 +125,12 @@ in {
networking.useNetworkd = true;
systemd.network.enable = true;
systemd.network.networks."30-wan" = {
matchConfig.Name = cfg.networkMatchConfigName;
matchConfig = networkMatchConfig;
networkConfig.DHCP = "no";
address = [
"${cfg.ipv4}/32"
"${cfg.ipv6}::/64"
];
] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
routes = [
{ Gateway = "172.31.1.1"; GatewayOnLink = true; }
{ Gateway = "fe80::1"; }
@@ -137,6 +172,13 @@ in {
};
};
};
assertions = [
{
assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
}
];
}
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
boot.initrd.kernelModules = [ "virtio_gpu" ];
+134
View File
@@ -0,0 +1,134 @@
{ ... }:
{
lib,
config,
...
}: let
cfg = config.hectic.hardware.njalla;
in {
options.hectic.hardware.njalla = {
enable = lib.mkEnableOption "Enable njalla hardware configurations";
ipv4 = lib.mkOption {
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
example = "185.193.126.103";
description = ''
Njalla IPv4 address assigned to the host.
'';
};
ipv4PrefixLength = lib.mkOption {
type = lib.types.int;
default = 24;
example = 24;
description = ''
Njalla IPv4 prefix length.
'';
};
ipv4Gateway = lib.mkOption {
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
default = "185.193.126.1";
example = "185.193.126.1";
description = ''
Njalla IPv4 gateway.
'';
};
ipv6 = lib.mkOption {
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
example = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
description = ''
Njalla IPv6 address assigned to the host.
'';
};
ipv6PrefixLength = lib.mkOption {
type = lib.types.int;
default = 64;
example = 64;
description = ''
Njalla IPv6 prefix length.
'';
};
ipv6Gateway = lib.mkOption {
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
default = "2a0a:3840:1337:126::1";
example = "2a0a:3840:1337:126::1";
description = ''
Njalla IPv6 gateway.
'';
};
networkMatchConfigName = lib.mkOption {
type = lib.types.str;
default = "eth0";
example = "eth0";
description = ''
Njalla container network interface name.
'';
};
device = lib.mkOption {
type = lib.types.str;
default = "/dev/vda";
example = "/dev/disk/by-id/virtio-root";
description = ''
Njalla installation disk for disko/nixos-anywhere.
`/dev/vda` is the default block device visible on the inspected Njalla
host. Prefer a stable `/dev/disk/by-id/...` path when available.
'';
};
enableDisko = lib.mkOption {
type = lib.types.bool;
default = true;
description = ''
Whether to provide a disko layout for nixos-anywhere installs.
'';
};
};
config = lib.mkIf cfg.enable (lib.mkMerge [
{
boot.isContainer = true;
networking.useDHCP = false;
networking.useNetworkd = true;
systemd.network.enable = true;
systemd.network.networks."30-wan" = {
matchConfig.Name = cfg.networkMatchConfigName;
networkConfig.DHCP = "no";
address = [
"${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
"${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
];
routes = [
{ Gateway = cfg.ipv4Gateway; }
{ Gateway = cfg.ipv6Gateway; }
];
};
networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
}
(lib.mkIf cfg.enableDisko {
boot.loader.grub.device = cfg.device;
disko.devices.disk.main = {
type = "disk";
device = cfg.device;
content = {
type = "gpt";
partitions = {
boot = {
size = "1M";
type = "EF02";
priority = 1;
};
root = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
};
};
};
};
})
]);
}
+1 -30
View File
@@ -25,36 +25,7 @@ in {
programs.bash.shellAliases.tmux = "tmux a";
home-manager.sharedModules = [
{
programs.tmux = {
enable = true;
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
keyMode = "vi";
escapeTime = 500;
historyLimit = 50000;
newSession = true;
extraConfig = ''
# resurrect
set -g @resurrect-strategy-vim 'session'
set -g @resurrect-strategy-nvim 'session'
set -g @resurrect-capture-pane-contents 'on'
resurrect_dir="$HOME/.tmux/resurrect"
set -g @resurrect-dir $resurrect_dir
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
# continuum
set -g @continuum-restore 'on'
set -g @continuum-boot 'on'
set -g @continuum-save-interval '10'
bind-key -T copy-mode-vi v send-keys -X begin-selection
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
bind-key O select-pane -t :.-
'';
};
}
(flake + "/home/module/program/tmux.nix")
];
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
+1 -1
View File
@@ -147,7 +147,7 @@ in {
(lib.mkIf cfg.watcher.enable {
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
sopsFile = "${flake}/sus/sentinella-default.yaml";
sopsFile = flake + "/sus/sentinella-default.yaml";
};
systemd.services."sentinella-watcher" = {
@@ -0,0 +1,153 @@
{ ... }: {
pkgs,
lib,
config,
...
}: let
cfg = config.hectic.services.stable-video-diffusion;
in {
options.hectic.services.stable-video-diffusion = {
enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
host = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
description = "Address the Stable Video Diffusion API binds to.";
};
port = lib.mkOption {
type = lib.types.port;
default = 7861;
description = "Port the Stable Video Diffusion API binds to.";
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.hectic.stable-video-diffusion-api;
defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
description = "Package providing the Stable Video Diffusion API executable.";
};
modelId = lib.mkOption {
type = lib.types.str;
default = "stabilityai/stable-video-diffusion-img2vid-xt";
description = "Model identifier loaded by the Stable Video Diffusion API.";
};
device = lib.mkOption {
type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
default = null;
description = ''
Torch device requested from the Stable Video Diffusion API. When null,
the package keeps its own auto-detection behavior.
'';
};
libraryPath = lib.mkOption {
type = with lib.types; listOf str;
default = [];
description = ''
Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
should include /run/opengl-driver/lib so libcuda.so is visible to torch.
'';
};
stateDir = lib.mkOption {
type = lib.types.str;
default = "/var/lib/stable-video-diffusion-api";
description = "Persistent state directory for the Stable Video Diffusion API.";
};
cacheDir = lib.mkOption {
type = lib.types.str;
default = "/var/cache/stable-video-diffusion-api";
description = "Cache directory for downloaded model and runtime artifacts.";
};
outputDir = lib.mkOption {
type = lib.types.str;
default = "${cfg.stateDir}/outputs";
defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
description = "Directory where generated video outputs are written.";
};
environmentFile = lib.mkOption {
type = with lib.types; nullOr path;
default = null;
description = ''
Optional environment file for secrets or runtime overrides. Values from
the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
and LD_LIBRARY_PATH by default.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Whether to open the API port in the firewall.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.device != "cuda" || cfg.libraryPath != [];
message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
}
];
users.users.stable-video-diffusion = {
isSystemUser = true;
group = "stable-video-diffusion";
};
users.groups.stable-video-diffusion = {};
systemd.tmpfiles.rules = [
"d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
"d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
"d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
];
systemd.services.stable-video-diffusion-api = {
description = "Stable Video Diffusion HTTP API";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
SVD_API_HOST = cfg.host;
SVD_API_PORT = toString cfg.port;
SVD_MODEL_ID = cfg.modelId;
SVD_STATE_DIR = cfg.stateDir;
SVD_CACHE_DIR = cfg.cacheDir;
SVD_OUTPUT_DIR = cfg.outputDir;
}
// lib.optionalAttrs (cfg.device != null) {
SVD_DEVICE = cfg.device;
}
// lib.optionalAttrs (cfg.libraryPath != []) {
LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
};
serviceConfig = lib.mkMerge [
{
Type = "simple";
User = "stable-video-diffusion";
Group = "stable-video-diffusion";
WorkingDirectory = cfg.stateDir;
ExecStart = lib.getExe' cfg.package "stable-video-diffusion-api";
Restart = "on-failure";
RestartSec = "5s";
TimeoutStopSec = "30s";
KillSignal = "SIGTERM";
KillMode = "mixed";
StandardOutput = "journal";
StandardError = "journal";
}
(lib.mkIf (cfg.environmentFile != null) {
EnvironmentFile = cfg.environmentFile;
})
];
};
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
};
}
@@ -1,230 +0,0 @@
# Matrix Cluster Failover Runbook (`accord.tube`)
Primary: `hectic-lab` (NL, `128.140.75.58`)
Standby: `bfs.poland.xray` (PL, `91.198.166.181`)
Module: `hectic.generic.matrix-cluster` (`nixos/module/generic/matrix-cluster.nix`).
Shared secrets: `sus/matrix-cluster.yaml`.
All `psql` and `pg_ctl` invocations use PostgreSQL **17** at data dir
`/var/lib/postgresql/17`.
## Initial setup
### 1. Provision shared SOPS file (`sus/matrix-cluster.yaml`)
On a workstation with both yukkop and yukkop-alt age keys available:
```sh
sudo cat /var/lib/matrix-synapse/homeserver.signing.key # on NL (hectic-lab)
# Copy the single line value into the buffer for the next step.
sops sus/matrix-cluster.yaml
```
Populate the editor with:
```yaml
matrix:
signing-key: <paste verbatim signing-key line from NL>
postgres-replication-password: <openssl rand -base64 32>
object-storage:
credentials: |
ACCESS_KEY_ID=<verbatim copy from sus/hectic-lab.yaml>
SECRET_ACCESS_KEY=<verbatim copy from sus/hectic-lab.yaml>
porkbun-api-key: <PORKBUN_API_KEY>
porkbun-secret-api-key: <PORKBUN_SECRET_API_KEY>
```
Verify recipients:
```sh
sops updatekeys sus/matrix-cluster.yaml
sops -d sus/matrix-cluster.yaml | grep -E 'signing-key|porkbun-api-key|object-storage'
```
Expected: all five keys present, exit 0.
### 2. Deploy NL primary first
```sh
nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux' --target-host root@128.140.75.58
```
Verify on NL:
```sh
sudo systemctl status matrix-synapse postgresql matrix-cluster-replication-password
sudo -u postgres psql -c "select rolname, rolreplication from pg_roles where rolname='replication';"
# Expected: replication | t
```
### 3. Seed PL replica with `pg_basebackup`
On PL:
```sh
sudo systemctl stop postgresql
sudo rm -rf /var/lib/postgresql/17
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
pg_basebackup \
-h 128.140.75.58 \
-p 5432 \
-U replication \
-D /var/lib/postgresql/17 \
-Fp -Xs -P -R \
--no-password
```
`-R` writes `standby.signal` and an initial `primary_conninfo`. The
matrix-cluster module's `matrix-cluster-standby-bootstrap` service will
overwrite `primary_conninfo` to use a libpq passfile on next boot.
### 4. Deploy PL standby
```sh
nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux' --target-host root@91.198.166.181
sudo systemctl start postgresql
```
Verify streaming on NL:
```sh
sudo -u postgres psql -c 'select client_addr, state, sync_state from pg_stat_replication;'
# Expected: 91.198.166.181 | streaming | async
```
Verify standby on PL:
```sh
sudo -u postgres psql -c 'select pg_is_in_recovery();'
# Expected: t
sudo systemctl is-active matrix-synapse
# Expected: inactive (standby keeps Synapse off)
```
### 5. Remove duplicate S3 credentials from `sus/hectic-lab.yaml`
Only AFTER NL is confirmed healthy reading from the new shared file:
```sh
sops sus/hectic-lab.yaml
# Delete the matrix/object-storage/credentials block.
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
```
## Normal operations
```sh
# NL: replication health
sudo -u postgres psql -c 'select * from pg_stat_replication;'
# Expected: 1 row, state=streaming, sync_state=async
# PL: replay status
sudo -u postgres psql -c 'select now() - pg_last_xact_replay_timestamp() as lag;'
# Both: cert renewal
sudo systemctl status acme-accord.tube.timer
sudo journalctl -u acme-accord.tube.service --since '24 hours ago'
# Synapse health (NL primary)
curl -sf https://accord.tube/_matrix/client/versions | head
```
## Planned failover (NL -> PL)
```sh
# 1. Drain NL: stop accepting writes.
sudo systemctl stop matrix-synapse
sudo systemctl stop postgresql # ensure no new WAL after this point
# 2. Promote PL replica.
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
# Wait until pg_is_in_recovery() returns f:
sudo -u postgres psql -c 'select pg_is_in_recovery();'
# 3. Make the role switch declarative before rebuilding.
# Edit the flake so rebuilds match the promoted database state:
# - nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
# hectic.generic.matrix-cluster.role = "primary";
# hectic.generic.matrix-cluster.overrideEnableSynapse = true;
# hectic.generic.matrix-cluster.secretsFile = config.sops.secrets."matrix/secrets".path;
# - nixos/system/hectic-lab/hectic-lab.nix:
# hectic.generic.matrix-cluster.role = "standby";
# hectic.generic.matrix-cluster.overrideEnableSynapse = false;
# hectic.generic.matrix-cluster.replication.peerHost = "91.198.166.181";
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "128.140.75.58/32" ];
# (You will also need a matrix/secrets entry on PL - copy from NL via SOPS.)
sudo nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux'
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
sudo systemctl status matrix-synapse
# 4. Swap DNS A record at Porkbun:
# accord.tube A 91.198.166.181 (was 128.140.75.58)
# TTL: set to 300 in advance of any planned failover.
# Porkbun UI: https://porkbun.com/account/domainsSpeedy -> accord.tube -> DNS -> edit A record.
# Or via API:
sudo curl -sX POST https://api.porkbun.com/api/json/v3/dns/editByNameType/accord.tube/A \
-H 'content-type: application/json' \
-d "$(jq -n --arg k "$PORKBUN_API_KEY" --arg s "$PORKBUN_SECRET_API_KEY" \
'{secretapikey:$s,apikey:$k,content:"91.198.166.181",ttl:"300"}')"
# 5. Federation smoke test.
curl -s 'https://federationtester.matrix.org/api/report?server_name=accord.tube' | jq .FederationOK
# Expected: true
```
Expected after the rebuilds:
- `bfs.poland.xray` evaluates and runs as `role = "primary"`.
- `hectic-lab` evaluates as `role = "standby"` with Synapse forced off.
- Future `nixos-rebuild` runs preserve the promoted topology instead of reapplying standby settings to PL.
## Failback (PL -> NL)
```sh
# 1. Stop NL postgres if still up; clear its data dir.
sudo systemctl stop postgresql matrix-synapse
sudo rm -rf /var/lib/postgresql/17
# 2. Re-seed NL from PL (now the live primary).
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
pg_basebackup -h 91.198.166.181 -p 5432 -U replication \
-D /var/lib/postgresql/17 -Fp -Xs -P -R --no-password
# 3. Temporarily flip roles in the flake:
# - hectic-lab.nix: role = "standby"; peerHost = "91.198.166.181";
# - bfs.poland.xray.nix: role = "primary"; peerHost = "128.140.75.58";
# Rebuild both.
# 4. Once NL is streaming green, do the reverse failover dance:
sudo systemctl stop matrix-synapse # on PL
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote # on NL
# Then revert the flake role assignments back to NL=primary / PL=standby and
# rebuild both hosts.
# 5. Swap DNS back at Porkbun (A -> 128.140.75.58).
```
## Disaster recovery (NL permanently lost)
```sh
# 1. Promote PL as the new permanent primary.
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
# 2. Edit nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
# hectic.generic.matrix-cluster.role = "primary";
# hectic.generic.matrix-cluster.overrideEnableSynapse = lib.mkForce null;
# hectic.generic.matrix-cluster.replication.peerHost = "<new-standby-ip>";
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "<new-standby-ip>/32" ];
# 3. Provision a new host (any region with Porkbun-managed DNS) and import
# self.nixosModules.matrix-cluster with role = "standby" pointed at PL's IP.
# 4. Bootstrap the new standby via pg_basebackup from PL exactly as in
# "Initial setup" step 3, replacing 128.140.75.58 with PL's IP.
# 5. Update Porkbun A record to PL's IP permanently.
```
@@ -9,16 +9,11 @@
config,
...
}: let
matrixBackend = "https://128.140.75.58";
matrixHost = "accord.tube";
jitsiHost = "meet.accord.tube";
elementEntryDomain = "element.bfs.band";
polandEntryDomain = "bfs.band";
backendProxyConfig = ''
proxy_ssl_server_name on;
proxy_ssl_name ${matrixHost};
proxy_set_header Host ${matrixHost};
'';
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
in {
imports = [
self.nixosModules.xray-system
@@ -46,7 +41,6 @@ in {
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
};
replication = {
peerHost = "128.140.75.58";
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
};
acme = {
@@ -71,6 +65,14 @@ in {
hostName = jitsiHost;
};
# NOTE(yukkop): disk was provisioned outside disko, so the expected partition
# label does not exist. Pin root to the live filesystem UUID so stage 1 can
# mount `/` reliably.
fileSystems."/" = lib.mkForce {
device = "/dev/disk/by-uuid/06b48ef1-a1eb-428d-821c-90c96a624542";
fsType = "ext4";
};
security.acme = {
acceptTerms = true;
defaults.email = "security@bfs.band";
@@ -124,19 +126,17 @@ in {
};
locations."= /livekit/jwt" = {
proxyPass = "${matrixBackend}/livekit/jwt";
extraConfig = backendProxyConfig;
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
};
locations."^~ /livekit/jwt/" = {
proxyPass = "${matrixBackend}/livekit/jwt/";
extraConfig = backendProxyConfig;
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
};
locations."= /livekit/sfu" = {
proxyPass = "${matrixBackend}/livekit/sfu";
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
proxyWebsockets = true;
extraConfig = backendProxyConfig + ''
extraConfig = ''
proxy_send_timeout 120;
proxy_read_timeout 120;
proxy_buffering off;
@@ -147,9 +147,9 @@ in {
};
locations."^~ /livekit/sfu/" = {
proxyPass = "${matrixBackend}/livekit/sfu/";
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
proxyWebsockets = true;
extraConfig = backendProxyConfig + ''
extraConfig = ''
proxy_send_timeout 120;
proxy_read_timeout 120;
proxy_buffering off;
@@ -160,13 +160,14 @@ in {
};
locations."^~ /_matrix/" = {
proxyPass = "${matrixBackend}/_matrix/";
extraConfig = backendProxyConfig;
proxyPass = "http://127.0.0.1:8008";
extraConfig = ''
client_max_body_size ${config.hectic.generic.matrix-cluster.maxUploadSize};
'';
};
locations."^~ /_synapse/client/" = {
proxyPass = "${matrixBackend}/_synapse/client/";
extraConfig = backendProxyConfig;
proxyPass = "http://127.0.0.1:8008";
};
};
@@ -203,41 +204,41 @@ in {
key = "matrix/signing-key";
owner = "matrix-synapse";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
sops.secrets."matrix/postgres-replication-password" = {
key = "matrix/postgres-replication-password";
owner = "postgres";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
sops.secrets."matrix/object-storage/credentials" = {
key = "matrix/object-storage/credentials";
owner = "matrix-synapse";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
sops.secrets."matrix/secrets" = {
key = "matrix/secrets";
owner = "matrix-synapse";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
sops.secrets."matrix/turn-secret" = {
key = "matrix/turn-secret";
owner = "turnserver";
group = "turnserver";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
sops.secrets."matrix/porkbun-api-key" = {
key = "matrix/porkbun-api-key";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
sops.secrets."matrix/porkbun-secret-api-key" = {
key = "matrix/porkbun-secret-api-key";
mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml";
sopsFile = matrixClusterSopsFile;
};
}
@@ -0,0 +1,74 @@
{ pkgs, ... }:
let
port = 22222;
stateDir = "/var/lib/experimental-sshd";
configFile = pkgs.writeText "experimental-sshd_config" ''
Port 22222
ListenAddress 0.0.0.0
ListenAddress ::
HostKey ${stateDir}/ssh_host_ed25519_key
HostKey ${stateDir}/ssh_host_rsa_key
PasswordAuthentication no
KbdInteractiveAuthentication no
PubkeyAuthentication yes
PermitRootLogin prohibit-password
UsePAM yes
AuthenticationMethods publickey
AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u
LogLevel DEBUG3
VersionAddendum none
HostKeyAlgorithms rsa-sha2-512,rsa-sha2-256,ssh-ed25519
KexAlgorithms curve25519-sha256,diffie-hellman-group14-sha256
Ciphers aes256-ctr,aes128-ctr
MACs hmac-sha2-512,hmac-sha2-256
'';
keygenScript = pkgs.writeShellScript "experimental-sshd-keygen" ''
set -eu
${pkgs.coreutils}/bin/mkdir -p -m 0700 ${stateDir}
if [ ! -f ${stateDir}/ssh_host_ed25519_key ]; then
${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f ${stateDir}/ssh_host_ed25519_key -N ""
fi
if [ ! -f ${stateDir}/ssh_host_rsa_key ]; then
${pkgs.openssh}/bin/ssh-keygen -t rsa -b 4096 -f ${stateDir}/ssh_host_rsa_key -N ""
fi
'';
in
{
environment.etc."ssh/experimental-sshd_config".source = configFile;
networking.firewall.allowedTCPPorts = [ port ];
systemd.services.experimental-sshd-keygen = {
description = "Generate experimental SSH host keys";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
StateDirectory = "experimental-sshd";
ExecStart = keygenScript;
};
};
systemd.services.experimental-sshd = {
description = "Experimental SSH daemon";
wantedBy = [ "multi-user.target" ];
wants = [ "experimental-sshd-keygen.service" ];
after = [ "network.target" "experimental-sshd-keygen.service" ];
serviceConfig = {
Type = "simple";
StateDirectory = "experimental-sshd";
RuntimeDirectory = "experimental-sshd";
ExecStart = "${pkgs.openssh}/bin/sshd -D -e -f /etc/ssh/experimental-sshd_config";
};
preStart = ''
${pkgs.openssh}/bin/sshd -t -f /etc/ssh/experimental-sshd_config
'';
};
}
+11 -4
View File
@@ -15,7 +15,7 @@ with builtins;
with lib;
let
domain = "hectic-lab.com";
matrixDomain = "accord.tube";
sshPort = 22;
mailUserNames = [
"security"
"founders"
@@ -57,6 +57,7 @@ in {
(import ./attic.nix { inherit flake self inputs domain; })
(import ./containers.nix { inherit flake self inputs; })
./experimental-sshd.nix
(import ./ente.nix { inherit domain; })
(import ./mechabellum.nix { inherit flake self inputs domain; })
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
@@ -80,6 +81,7 @@ in {
enable = true;
networkMatchConfigName = "enp1s0";
ipv4 = "128.140.75.58";
floatingIpv4 = "78.47.243.0";
ipv6 = "2a01:4f8:c2c:d54a";
};
services.matrix = {
@@ -106,6 +108,7 @@ in {
'';
environment.systemPackages = with pkgs; [
tcpdump
git
rsync
python311
@@ -116,7 +119,7 @@ in {
sops = {
gnupg.sshKeyPaths = [ ];
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
defaultSopsFile = "${flake}/sus/hectic-lab.yaml";
defaultSopsFile = flake + "/sus/hectic-lab.yaml";
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
"init-postgresql" = {
key = "init-postgresql";
@@ -133,6 +136,8 @@ in {
};
users.users.root.openssh.authorizedKeys.keys = [
# neuro machine
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
# yukkop
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
@@ -159,6 +164,8 @@ in {
];
};
services.openssh.ports = [ sshPort ];
services.mailserver = {
enable = true;
domain = domain;
@@ -179,10 +186,10 @@ in {
networking.firewall = {
allowedTCPPorts = [
sshPort # ssh
80
443
3306 # mysql
11012 # gitea ssh
25565
55228 # ss-bfs
];
@@ -264,7 +271,7 @@ in {
settings.service.DISABLE_REGISTRATION = true;
settings.server = {
HTTP_PORT = 11011;
#SSH_PORT = 22;
SSH_PORT = sshPort;
SSH_DOMAIN = "hectic-lab.com";
};
database = {
+1
View File
@@ -37,6 +37,7 @@ in {
proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
extraConfig = ''
proxy_http_version 1.1;
client_max_body_size 500M;
'';
};
+2 -22
View File
@@ -15,32 +15,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
self.overlays.default
inputs.nix-minecraft.overlay
];
config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
config.allowUnfreePredicate = pkg:
self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
"minecraft-server"
"neoforge"
"nvidia-x11"
"cuda_nvcc"
"cuda_cudart"
"cuda_cuobjdump"
"cuda_cupti"
"cuda_nvdisasm"
"cuda_cccl"
"cuda_nvml_dev"
"cuda_nvrtc"
"cuda_nvtx"
"cuda_profiler_api"
"libcusparse_lt"
"libcublas"
"libcufft"
"libcufile"
"libcurand"
"libcusolver"
"libnvjitlink"
"libcusparse"
"cudnn"
];
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
config.permittedInsecurePackages = [
+18
View File
@@ -17,6 +17,11 @@
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
pkgs.stdenv.cc.cc.lib
pkgs.zlib
];
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
pname = "ollama";
version = "0.22.1";
@@ -176,6 +181,19 @@ in {
openFirewall = false;
};
hectic.services.stable-video-diffusion = {
enable = true;
host = "127.0.0.1";
port = 7861;
package = pkgs.hectic.stable-video-diffusion-api;
device = "cuda";
libraryPath = [
stableVideoDiffusionLibraryPath
"/run/opengl-driver/lib"
];
openFirewall = false;
};
networking = {
networkmanager.enable = true;
useDHCP = lib.mkDefault true;
+20
View File
@@ -0,0 +1,20 @@
{
flake,
self,
inputs,
system ? "x86_64-linux",
...
}: let
# Use folder name as name of this system
name = builtins.baseNameOf ./.;
in self.lib.nixpkgs-lib.nixosSystem {
pkgs = import inputs.nixpkgs {
inherit system;
overlays = [ self.overlays.default ];
};
modules = [
{ networking.hostName = name; }
(import ./${name}.nix { inherit flake self inputs; })
];
}
+35
View File
@@ -0,0 +1,35 @@
{
inputs,
self,
...
}: {
pkgs,
modulesPath,
...
}: {
imports = [
(modulesPath + "/profiles/qemu-guest.nix")
inputs.sops-nix.nixosModules.sops
self.nixosModules.hectic
];
hectic = {
archetype.dev.enable = true;
hardware.hetzner-cloud = {
enable = true;
device = "/dev/sda";
networkMatchConfigMac = "92:00:08:4a:b0:32";
ipv4 = "46.225.237.218";
ipv6 = "2a01:4f8:c2c:3b14";
};
};
users.users.root.openssh.authorizedKeys.keys = [
''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
];
environment.systemPackages = with pkgs; [
git
rsync
];
}
+13 -8
View File
@@ -1,18 +1,21 @@
# db-tool
PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries.
PostgreSQL utility package exposing separate development and operations binaries.
`db-dev` preserves the current development workflow via the `database` binary;
`db-ops` provides target-safe operational helpers such as secrets hydration.
## Provided Binaries
| Binary | Description |
| --- | --- |
| `database` | Main script for managing migrations, deployments, and logs. |
| `database` | Main `db-dev` script for managing local development databases, migrations, deployments, and logs. |
| `db-ops` | Operational helper binary for target/runtime database actions. |
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
## Required Environment Variables
These variables must be set for `db-tool` to function.
These variables must be set for `db-dev` / `database` to function.
| Variable | Description |
| --- | --- |
@@ -39,7 +42,7 @@ These variables must be set for `db-tool` to function.
## Postgres Package Override
By default, `db-tool`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
By default, `db-dev`/`db-ops`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
```nix
let
@@ -48,7 +51,8 @@ let
]);
in {
packages = [
(pkgs.hectic."db-tool".override { postgresql = myPg; })
(pkgs.hectic."db-dev".override { postgresql = myPg; })
(pkgs.hectic."db-ops".override { postgresql = myPg; })
(pkgs.hectic."postgres-init".override { postgresql = myPg; })
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
];
@@ -102,14 +106,14 @@ used by `database restore`.
## shellHook Example
To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
To use `db-dev` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
```nix
{
# ...
devShells.default = pkgs.mkShell {
packages = [
pkgs.hectic.db-tool
pkgs.hectic.db-dev
pkgs.hectic.postgres-init
pkgs.hectic.postgres-cleanup
];
@@ -134,7 +138,7 @@ To use `db-tool` in a Nix development shell, add the following to your `flake.ni
## hectic Bundle
`db-tool` and `migrator` apply a single bundle of SQL files that bootstrap the
`db-dev`, `db-ops`, and `migrator` apply a single bundle of SQL files that bootstrap the
`hectic` schema. The bundle lives in
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
contract, file layout, and the `self.lib.hectic.*` Nix API.
@@ -190,6 +194,7 @@ ALTER DATABASE mydb SET hectic.inheritance_extra_excluded_schemas = 'legacy,etl'
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. |
The bundle is idempotent — repeated application is safe.
+222
View File
@@ -0,0 +1,222 @@
# shellcheck shell=dash
: "${SCRIPT_NAME:=$(basename "$0")}"
help() {
# shellcheck disable=SC2059
printf "$(cat <<EOF
${BGREEN}Usage:${NC} $SCRIPT_NAME [OPTIONS] <SUBCOMMAND> [OPTIONS]
PostgreSQL operations utility.
${BGREEN}Global Options:${NC}
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
${BGREEN}Subcommands:${NC}
${BCYAN}secrets load${NC} [OPTIONS] Apply hectic bundle and load secrets
${BGREEN}Environment:${NC}
${BBLACK}PGURL${NC} PostgreSQL connection string fallback
${BBLACK}DB_URL${NC} PostgreSQL connection string fallback
${BBLACK}HECTIC_DOTENV_CONTENT${NC} Raw dotenv content to load
${BBLACK}HECTIC_DOTENV_FILE${NC} Dotenv file to read when readable
${BBLACK}LOCAL_DIR${NC} Used with ENVIRONMENT for .env fallback
${BBLACK}ENVIRONMENT${NC} Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
EOF
)"
}
help_secrets_load() {
# shellcheck disable=SC2059
printf "$(cat <<EOF
${BGREEN}Usage:${NC} $SCRIPT_NAME ${BCYAN}secrets load${NC} [OPTIONS]
Apply the hectic SQL bundle and load dotenv-backed secrets into
${BBLACK}hectic.secret${NC}.
${BGREEN}Options:${NC}
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} <path> Dotenv file path
${BGREEN}Resolution order:${NC}
1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC}
2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC}
3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv
source can be resolved.
EOF
)"
}
resolve_pgurl() {
if [ -n "${PGURL:-}" ]; then
printf '%s' "$PGURL"
elif [ -n "${DB_URL:-}" ]; then
printf '%s' "$DB_URL"
else
return 1
fi
}
resolve_dotenv_content() {
dotenv_file="${1:-}"
if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then
printf '%s' "$HECTIC_DOTENV_CONTENT"
elif [ -n "$dotenv_file" ]; then
if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then
return 2
fi
cat "$dotenv_file"
elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
cat "${LOCAL_DIR}/.env.${ENVIRONMENT}"
else
return 1
fi
}
subcommand_secrets_load() {
change_namespace 'db ops secrets load'
dotenv_file="${HECTIC_DOTENV_FILE:-}"
while [ $# -gt 0 ]; do
case $1 in
-h|--help)
help_secrets_load
restore_namespace
exit 0
;;
-u|--url)
if [ $# -lt 2 ]; then
log error "missing value for $1"
restore_namespace
exit 3
fi
PGURL=$2
DB_URL=$2
shift 2
;;
-f|--dotenv-file)
if [ $# -lt 2 ]; then
log error "missing value for $1"
restore_namespace
exit 3
fi
dotenv_file=$2
shift 2
;;
--*|-*)
log error "secrets load argument $1 does not exist"
restore_namespace
exit 9
;;
*)
log error "secrets load subcommand $1 does not exist"
restore_namespace
exit 9
;;
esac
done
if ! pgurl="$(resolve_pgurl)"; then
log error "PGURL or DB_URL is required"
restore_namespace
exit 3
fi
if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then
:
else
dotenv_content_exit_code=$?
if [ "$dotenv_content_exit_code" -eq 2 ]; then
log error "dotenv file is not readable: $dotenv_file"
else
log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)"
fi
restore_namespace
exit 3
fi
log notice "apply hectic bundle and load secrets"
apply_hectic_bundle "$pgurl" "$dotenv_content"
restore_namespace
}
subcommand_secrets() {
change_namespace 'db ops secrets'
if [ $# -eq 0 ]; then
help_secrets_load
restore_namespace
exit 0
fi
subcommand=$1
shift
case $subcommand in
load)
subcommand_secrets_load "$@"
;;
-h|--help)
help_secrets_load
restore_namespace
exit 0
;;
*)
log error "secrets subcommand $subcommand does not exist"
restore_namespace
exit 9
;;
esac
}
while [ $# -gt 0 ]; do
case $1 in
-h|--help)
help
exit 0
;;
-u|--url)
if [ $# -lt 2 ]; then
log error "missing value for $1"
exit 3
fi
PGURL=$2
DB_URL=$2
shift 2
;;
--*|-*)
log error "argument $1 does not exist"
exit 9
;;
*)
break
;;
esac
done
subcommand="${1:-}"
if [ -z "$subcommand" ]; then
help
exit 0
fi
shift
case $subcommand in
secrets)
subcommand_secrets "$@"
;;
*)
log error "subcommand $subcommand does not exist"
exit 9
;;
esac
+29 -3
View File
@@ -8,7 +8,7 @@ let
applyBundle = self.lib.hectic.applyBundleScript;
mkDatabase =
mkDbDev =
{ postgresql ? postgresql_17 }:
hectic.writeShellApplication {
inherit shell;
@@ -27,7 +27,7 @@ let
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
${applyBundle}
${builtins.readFile ./database.sh}
${builtins.readFile ./db-dev.sh}
'';
meta = {
@@ -36,6 +36,31 @@ let
};
};
mkDbOps =
{ postgresql ? postgresql_17 }:
hectic.writeShellApplication {
inherit shell;
bashOptions = [
"errexit"
"nounset"
];
excludeShellChecks = [ "SC2209" ];
name = "db-ops";
runtimeInputs = [ postgresql coreutils ];
text = ''
${builtins.readFile hectic.helpers.posix-shell.log}
${builtins.readFile hectic.helpers.posix-shell.change_namespace}
${applyBundle}
${builtins.readFile ./db-ops.sh}
'';
meta = {
description = "PostgreSQL operations utility";
mainProgram = "db-ops";
};
};
mkPostgresInit =
{ postgresql ? postgresql_17 }:
hectic.writeShellApplication {
@@ -73,7 +98,8 @@ let
};
in
{
"db-tool" = lib.makeOverridable mkDatabase { };
"db-dev" = lib.makeOverridable mkDbDev { };
"db-ops" = lib.makeOverridable mkDbOps { };
"postgres-init" = lib.makeOverridable mkPostgresInit { };
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
"hectic-inheritance" = hecticInheritance;
+37
View File
@@ -1,9 +1,46 @@
#!/bin/dash
# Stop a local PostgreSQL cluster started by postgres-init.
#
# Public contract:
# - Accepts PG_WORKING_DIR directly, or derives it from LOCAL_DIR.
# - If no cluster root can be resolved, exits successfully without doing
# anything. This keeps cleanup safe in traps and partial-init flows.
# - If no postmaster.pid exists, treats the cluster as already stopped.
# - NO_TTY=1 redirects pg_ctl output into a temp log file instead of writing to
# the current terminal.
postgres_cleanup_help() {
cat <<'EOF'
Usage: postgres-cleanup
Stop a local PostgreSQL cluster if it is running.
Environment:
PG_WORKING_DIR Cluster root directory
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
NO_TTY Redirect pg_ctl output to a temp file
Behavior:
- Returns success if the cluster directory cannot be resolved.
- Returns success if postmaster.pid is already absent.
- Ignores pg_ctl stop errors so cleanup remains trap-safe.
EOF
}
postgres_cleanup_main() {
case "${1:-}" in
-h|--help)
postgres_cleanup_help
return 0
;;
esac
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
# Cleanup is intentionally forgiving so it can be used in traps after
# partial startup failures without masking the real error.
if [ "${NO_TTY:-0}" = "1" ]; then
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
+155 -5
View File
@@ -1,6 +1,58 @@
#!/bin/dash
# Initialize or reuse a local PostgreSQL cluster for development workflows.
#
# Public contract:
# - Requires either PG_WORKING_DIR or LOCAL_DIR.
# - Produces a ready-to-use database and exports:
# POSTGRESQL_HOST, POSTGRESQL_PORT, POSTGRESQL_USER, POSTGRESQL_DATABASE,
# PGURL, and also the variable named by PG_URL_VAR.
# - Reuses an existing cluster only when PG_REUSE is set and PG_VERSION exists.
# - If a reused cluster cannot start, it is reinitialized automatically.
#
# Important knobs:
# - PG_WORKING_DIR: cluster root (defaults to $LOCAL_DIR/focus/postgresql)
# - PG_DATABASE: database to create/ensure (default: testdb)
# - PG_PORT: socket port (default: 5432)
# - PG_URL_VAR: alternate URL variable to export in addition to PGURL
# - PG_CONF_FILE: base postgresql.conf to copy on fresh init
# - PG_SHARED_PRELOAD_LIBRARIES: preload list; empty string disables pg_cron
# - PG_DISABLE_LOGGING=1: disable logging_collector in generated config
# - NO_TTY=1: redirect pg_ctl stop/start output into temp log files
postgres_init_help() {
cat <<'EOF'
Usage: postgres-init
Initialize or reuse a local PostgreSQL cluster.
Environment:
PG_WORKING_DIR Cluster root directory
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
PG_DATABASE Database name to create/ensure (default: testdb)
PG_PORT PostgreSQL port / unix socket port (default: 5432)
PG_URL_VAR Extra URL variable to export alongside PGURL
PG_CONF_FILE Base postgresql.conf copied on fresh init only
PG_SHARED_PRELOAD_LIBRARIES Preload list; empty disables pg_cron preload
PG_DISABLE_LOGGING Set to 1 to disable logging_collector
PG_REUSE Reuse existing cluster if PG_VERSION exists
NO_TTY Redirect pg_ctl output to temp files
Behavior:
- Rewrites runtime socket/port/cron settings on every launch.
- Creates the target database if missing.
- If a reused cluster exists but cannot start, reinitializes it automatically.
EOF
}
postgres_init_main() {
case "${1:-}" in
-h|--help)
postgres_init_help
return 0
;;
esac
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
return 1
@@ -25,20 +77,41 @@ postgres_init_main() {
fi
mkdir -p "$sockdir" || return 1
# Reuse is optimistic: if a cluster exists on disk, try to start it first.
# We only destroy state after a real startup failure proves the cluster is
# broken, which keeps long-lived local DBs intact when possible.
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
if [ "$PG_REUSE" -eq 0 ]; then
rm -rf "$data" "$sockdir" || return 1
mkdir -p "$sockdir" || return 1
initdb -D "$data" --no-locale -E UTF8 || return 1
if [ -n "${PG_CONF_FILE:-}" ]; then
# PG_CONF_FILE replaces the base postgresql.conf on fresh init only. We
# still append runtime values later so the helper can relocate sockets,
# ports, and cron settings regardless of the custom file contents.
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
else
{ printf '%s\n' "listen_addresses = ''"; [ "$PG_DISABLE_LOGGING" -eq 0 ] && { printf '%s\n' 'logging_collector = on'; printf '%s\n' "log_directory = 'log'"; }; [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ] && { printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"; printf '%s\n' "cron.database_name = '$db'"; printf '%s\n' "cron.host = '$sockdir'"; }; :; } >> "$data/postgresql.conf" || return 1
{
printf '%s\n' "listen_addresses = ''"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' 'logging_collector = on'
printf '%s\n' "log_directory = 'log'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.conf" || return 1
fi
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
fi
# Rewrite the runtime knobs on every launch. Reused clusters may have been
# started from another workspace/session, so we must override stale socket,
# port, and pg_cron wiring before attempting startup.
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
@@ -68,12 +141,85 @@ postgres_init_main() {
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.auto.conf" || return 1
_pg_start_exit=0
if [ "${NO_TTY:-0}" = "1" ]; then
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || return 2
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
else
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || return 2
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
fi
if [ "$_pg_start_exit" -ne 0 ]; then
if [ "$PG_REUSE" -eq 1 ]; then
# Self-heal path: the on-disk cluster exists but cannot complete startup
# (for example bad WAL / invalid checkpoint). At this point preserving
# the broken state is less useful than reinitializing automatically.
printf '%s\n' "postgres-init: reused cluster failed to start; reinitializing $wd" >&2
PG_REUSE=0
rm -rf "$data" "$sockdir" || return 1
mkdir -p "$sockdir" || return 1
initdb -D "$data" --no-locale -E UTF8 || return 1
if [ -n "${PG_CONF_FILE:-}" ]; then
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
else
{
printf '%s\n' "listen_addresses = ''"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' 'logging_collector = on'
printf '%s\n' "log_directory = 'log'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.conf" || return 1
fi
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
{
printf '%s\n' "port = '$PG_PORT'"
printf '%s\n' "unix_socket_directories = '$sockdir'"
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' "logging_collector = 'on'"
else
printf '%s\n' "logging_collector = 'off'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.auto.conf" || return 1
_pg_start_exit=0
if [ "${NO_TTY:-0}" = "1" ]; then
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
else
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
fi
[ "$_pg_start_exit" -eq 0 ] || return 2
printf '%s\n' "postgres-init: reinitialized broken cluster at $wd" >&2
else
return 2
fi
fi
user="$(id -un)" || return 1
@@ -87,9 +233,13 @@ postgres_init_main() {
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
_pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
# Export both names for compatibility: some callers consume plain PGURL,
# while multi-cluster shells also need a project-specific variable like
# WIPE_PGURL or BMSEARCH_PGURL in the same environment.
PGURL="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
export PGURL
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
export "${PG_URL_VAR}=${_pg_url}" || return 1
export "${PG_URL_VAR}=${PGURL}" || return 1
return 0
}
+6 -1
View File
@@ -132,6 +132,7 @@ in {
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
migration-name = pkgs.callPackage ./migration-name.nix {};
plgo = pkgs.callPackage ./plgo {};
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
@@ -142,15 +143,18 @@ in {
watch = pkgs.callPackage ./c/watch/default.nix {};
support-bot = pkgs.callPackage ./support-bot {};
gitea-heatmap = pkgs.callPackage ./gitea {};
gitea-runner-nix-image = pkgs.callPackage ./gitea-runner-nix-image {};
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
deploy = pkgs.callPackage ./deploy { inherit inputs; };
element-web = pkgs.callPackage ./element-web {};
shellplot = pkgs.callPackage ./shellplot {};
which-country-rs = pkgs.callPackage ./which-country-rs {};
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
migrator = pkgs.callPackage ./migrator { inherit self; };
"parse-uri" = pkgs.callPackage ./parse-uri {};
"db-tool" = dbToolPkgs."db-tool";
"db-dev" = dbToolPkgs."db-dev";
"db-ops" = dbToolPkgs."db-ops";
"postgres-init" = dbToolPkgs."postgres-init";
"postgres-cleanup" = dbToolPkgs."postgres-cleanup";
"hectic-inheritance" = dbToolPkgs."hectic-inheritance";
@@ -172,5 +176,6 @@ in {
pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
pg-15-ext-plsh = buildPlShExt pkgs "15";
stable-video-diffusion-api = pkgs.callPackage ./stable-video-diffusion-api {};
media-browser = pkgs.callPackage ./media-browser {};
}
+1 -1
View File
@@ -36,7 +36,7 @@ let
hash = "sha256-pbzuPgKJ0DmrDSTO7ZTDArX+Xr9k/ndAGZvQg2kMTMQ=";
};
inherit patches;
#inherit patches; #WIP
pnpmDeps = fetchPnpmDeps {
pname = "element";
@@ -0,0 +1,81 @@
{
bash,
cacert,
coreutils,
dockerTools,
git,
lib,
nix,
symlinkJoin,
}:
let
name = "gitea-runner-nix-image";
tag = "2026-06-07";
root = symlinkJoin {
name = "${name}-root";
paths = [
bash
cacert
coreutils
git
nix
];
};
in
dockerTools.buildLayeredImageWithNixDb {
inherit name tag;
contents = [ root ];
fakeRootCommands = ''
mkdir -p ./etc
cat > ./etc/passwd <<'EOF'
root:x:0:0:root:/root:/bin/bash
nobody:x:65534:65534:nobody:/var/empty:/bin/false
EOF
cat > ./etc/group <<'EOF'
root:x:0:
nixbld:x:30000:
nobody:x:65534:
EOF
for n in $(seq 1 10); do
echo "nixbld$n:x:$((30000 + n)):30000:Nix build user $n:/var/empty:/bin/false" >> ./etc/passwd
sed -i "s/^nixbld:x:30000:.*/&,nixbld$n/" ./etc/group
done
mkdir -p ./nix/var/nix/{gcroots,profiles,temproots,userpool,db}
mkdir -p ./nix/var/log/nix/drvs
chmod 1777 ./nix/var/nix/gcroots ./nix/var/nix/profiles
'';
extraCommands = ''
mkdir -p etc/nix root tmp
chmod 1777 tmp
cat > etc/nix/nix.conf <<'EOF'
accept-flake-config = true
experimental-features = nix-command flakes
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
trusted-users = root
sandbox = false
EOF
'';
config = {
Cmd = [ "${bash}/bin/bash" ];
Env = [
"HOME=/root"
"PATH=${lib.makeBinPath [ bash coreutils git nix ]}"
"SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
"USER=root"
];
Labels = {
"org.opencontainers.image.description" = "Nix-capable Gitea Actions job image";
"org.opencontainers.image.ref.name" = "${name}:${tag}";
"org.opencontainers.image.source" = "https://gitea.hectic-lab.com/hectic-lab/util.nix";
};
WorkingDir = "/workspace";
};
}
+1
View File
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
Commands: []*cli.Command{
newUserCommand(),
newRepoSyncReleasesCommand(),
newHeatmapCommand(),
newRegenerateCommand(),
newAuthCommand(),
newSendMailCommand(),
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cmd
import (
"context"
"fmt"
"strings"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/urfave/cli/v3"
)
func newHeatmapCommand() *cli.Command {
return &cli.Command{
Name: "heatmap",
Usage: "Manage heatmap indexes",
Commands: []*cli.Command{
newHeatmapReindexCommand(),
},
}
}
func newHeatmapReindexCommand() *cli.Command {
return &cli.Command{
Name: "reindex",
Usage: "Reindex heatmap contributions from repository default branches",
Action: runHeatmapReindex,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "all",
Usage: "Reindex all repositories",
},
&cli.StringFlag{
Name: "repo",
Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
},
&cli.StringFlag{
Name: "owner",
Usage: "Owner name for --repo",
},
},
}
}
func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
if err := initDB(ctx); err != nil {
return err
}
if err := git.InitSimple(); err != nil {
return err
}
reindexAll := c.Bool("all")
repoName := c.String("repo")
ownerName := c.String("owner")
if reindexAll {
if repoName != "" || ownerName != "" {
return fmt.Errorf("--all cannot be combined with --repo or --owner")
}
return reindexAllHeatmapRepositories(ctx)
}
if repoName == "" {
return fmt.Errorf("either --all or --repo must be provided")
}
ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
if err != nil {
return err
}
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
if err != nil {
return err
}
return reindexHeatmapRepository(ctx, repo)
}
func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
if ownerName != "" {
if strings.Contains(repoName, "/") {
return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
}
return ownerName, repoName, nil
}
ownerName, repoName, ok := strings.Cut(repoName, "/")
if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
}
return ownerName, repoName, nil
}
func reindexAllHeatmapRepositories(ctx context.Context) error {
for page := 1; ; page++ {
repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
ListOptions: db.ListOptions{
PageSize: repo_model.RepositoryListDefaultPageSize,
Page: page,
},
Private: true,
})
if err != nil {
return fmt.Errorf("SearchRepositoryByName: %w", err)
}
if len(repos) == 0 {
break
}
log.Trace("Processing next %d repos of %d", len(repos), count)
for _, repo := range repos {
if err := reindexHeatmapRepository(ctx, repo); err != nil {
log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
continue
}
}
}
return nil
}
func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
}
return nil
}
@@ -0,0 +1,154 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cmd
import (
"fmt"
"strings"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/timeutil"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapAdminReindex(t *testing.T) {
repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
})
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
}
func TestHeatmapAdminRepoSelector(t *testing.T) {
owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
require.NoError(t, err)
assert.Equal(t, "user", owner)
assert.Equal(t, "repo", repo)
owner, repo, err = parseHeatmapRepoSelector("user", "repo")
require.NoError(t, err)
assert.Equal(t, "user", owner)
assert.Equal(t, "repo", repo)
_, _, err = parseHeatmapRepoSelector("", "repo")
assert.ErrorContains(t, err, "owner/name")
_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
assert.ErrorContains(t, err, "when --owner is set")
}
type heatmapAdminTestCommit struct {
Branch string
Mark string
Parent string
AuthorName string
AuthorEmail string
CommitterName string
CommitterEmail string
AuthorDate string
}
func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
t.Helper()
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
require.NoError(t, err)
marks := runHeatmapAdminFastImport(t, repo, commits)
repo.IsEmpty = false
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
return repo, marks
}
func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
t.Helper()
var stream strings.Builder
branchTips := make(map[string]string)
for i, commit := range commits {
mark := fmt.Sprintf(":%d", i+1)
branchRef := "refs/heads/" + commit.Branch
stream.WriteString("commit " + branchRef + "\n")
stream.WriteString("mark " + mark + "\n")
stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
message := "heatmap admin " + commit.Mark
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
if parentMark := branchTips[commit.Branch]; parentMark != "" {
stream.WriteString("from " + parentMark + "\n")
} else if commit.Parent != "" {
stream.WriteString("from " + commit.Parent + "\n")
}
content := commit.Mark + "\n"
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
branchTips[commit.Branch] = mark
}
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
WithDir(repo.RepoPath()).
WithStdinCopy(strings.NewReader(stream.String())).
Run(t.Context()))
commitSHAs := make(map[string]string, len(commits))
for _, commit := range commits {
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
AddDynamicArguments("refs/heads/" + commit.Branch).
WithDir(repo.RepoPath()).
RunStdString(t.Context())
require.NoError(t, err)
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
}
return commitSHAs
}
func heatmapAdminSignatureLine(kind, name, email, date string) string {
parsed, err := time.Parse(time.RFC3339, date)
if err != nil {
panic(err)
}
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
}
func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
t.Helper()
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
require.NoError(t, err)
return contributions
}
@@ -0,0 +1,121 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package activities
import (
"context"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/xorm/schemas"
)
// HeatmapContribution stores commit contributions counted for profile heatmaps.
type HeatmapContribution struct {
ID int64 `xorm:"pk autoincr"`
UserID int64 `xorm:"NOT NULL"`
RepoID int64 `xorm:"NOT NULL"`
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing.
type HeatmapContributionIdentity struct {
RepoID int64
CommitSHA string
UserID int64
}
func init() {
db.RegisterModel(new(HeatmapContribution))
}
// TableIndices implements xorm's TableIndices interface.
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
return []*schemas.Index{uniqueContribution, userAuthorRepo}
}
// UpsertHeatmapContribution creates or updates a commit contribution without duplicating counted rows.
func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContribution) error {
return db.WithTx(ctx, func(ctx context.Context) error {
e := db.GetEngine(ctx)
rows, err := e.Where("repo_id=? AND commit_sha=? AND user_id=?", contribution.RepoID, contribution.CommitSHA, contribution.UserID).
Cols("author_email", "author_unix").
Update(contribution)
if err != nil {
return err
}
if rows > 0 {
return nil
}
has, err := e.Exist(&HeatmapContribution{RepoID: contribution.RepoID, CommitSHA: contribution.CommitSHA, UserID: contribution.UserID})
if err != nil {
return err
}
if has {
return nil
}
_, err = e.Insert(contribution)
return err
})
}
// CountHeatmapContributions returns counted heatmap contribution rows for a user.
func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) {
return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution))
}
// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities.
func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error {
if len(identities) == 0 {
_, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution))
return err
}
current := make(map[HeatmapContributionIdentity]struct{}, len(identities))
for _, identity := range identities {
current[identity] = struct{}{}
}
contributions, err := FindHeatmapContributionsByRepo(ctx, repoID)
if err != nil {
return err
}
for _, contribution := range contributions {
identity := HeatmapContributionIdentity{
RepoID: contribution.RepoID,
CommitSHA: contribution.CommitSHA,
UserID: contribution.UserID,
}
if _, ok := current[identity]; ok {
continue
}
if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil {
return err
}
}
return nil
}
// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository.
func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) {
contributions := make([]*HeatmapContribution, 0)
return contributions, db.GetEngine(ctx).
Where("repo_id=?", repoID).
OrderBy("author_unix ASC, commit_sha ASC, user_id ASC").
Find(&contributions)
}
@@ -0,0 +1,81 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package activities_test
import (
"testing"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapContributionModel(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
assert.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
const (
userID = int64(2)
repoID = int64(1)
commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
authorUnix = timeutil.TimeStamp(1579089600)
)
contribution := &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: "user2@example.com",
AuthorUnix: authorUnix,
}
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), contribution))
count, err := activities_model.CountHeatmapContributions(t.Context(), userID)
require.NoError(t, err)
assert.EqualValues(t, 1, count)
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: "changed@example.com",
AuthorUnix: authorUnix + 900,
}))
count, err = activities_model.CountHeatmapContributions(t.Context(), userID)
require.NoError(t, err)
assert.EqualValues(t, 1, count)
stored := &activities_model.HeatmapContribution{UserID: userID, RepoID: repoID, CommitSHA: commitSHA}
has, err := db.GetEngine(t.Context()).Get(stored)
require.NoError(t, err)
require.True(t, has)
assert.Equal(t, "changed@example.com", stored.AuthorEmail)
assert.Equal(t, authorUnix+900, stored.AuthorUnix)
assert.NotZero(t, stored.CreatedUnix)
assert.NotZero(t, stored.UpdatedUnix)
err = db.Insert(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: "duplicate@example.com",
AuthorUnix: authorUnix,
})
assert.Error(t, err)
includePrivate, err := user_model.GetIncludePrivateContributions(t.Context(), userID)
require.NoError(t, err)
assert.False(t, includePrivate)
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), userID, true))
includePrivate, err = user_model.GetIncludePrivateContributions(t.Context(), userID)
require.NoError(t, err)
assert.True(t, includePrivate)
}
@@ -10,6 +10,7 @@ import (
"code.gitea.io/gitea/models/organization"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
)
@@ -38,35 +39,45 @@ func getUserHeatmapData(ctx context.Context, user *user_model.User, team *organi
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
groupBy := "created_unix / 900 * 900"
groupBy := "author_unix / 900 * 900"
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
switch {
case setting.Database.Type.IsMySQL():
groupBy = "created_unix DIV 900 * 900"
groupBy = "author_unix DIV 900 * 900"
case setting.Database.Type.IsMSSQL():
groupByName = groupBy
}
cond, err := ActivityQueryCondition(ctx, GetFeedsOptions{
RequestedUser: user,
RequestedTeam: team,
Actor: doer,
IncludePrivate: true, // don't filter by private, as we already filter by repo access
IncludeDeleted: true,
// * Heatmaps for individual users only include actions that the user themself did.
// * For organizations actions by all users that were made in owned
// repositories are counted.
OnlyPerformedBy: !user.IsOrganization(),
})
includePrivate, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
if err != nil {
return nil, err
}
return hdata, db.GetEngine(ctx).
Select(groupBy+" AS timestamp, count(user_id) as contributions").
Table("action").
Where(cond).
And("created_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
sess := db.GetEngine(ctx).
Select(groupBy+" AS timestamp, count(heatmap_contribution.user_id) as contributions").
Table("heatmap_contribution").
Join("INNER", "repository", "repository.id = heatmap_contribution.repo_id").
Join("INNER", "`user` AS repo_owner", "repo_owner.id = repository.owner_id").
Where("author_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
And("author_unix <= ?", timeutil.TimeStampNow())
if !includePrivate {
sess = sess.And("repository.is_private = ?", false).
And("repo_owner.visibility = ?", structs.VisibleTypePublic)
}
if user.IsOrganization() {
sess = sess.And("repository.owner_id = ?", user.ID)
if team != nil && !team.IncludesAllRepositories {
sess = sess.Join("INNER", "team_repo", "team_repo.repo_id = repository.id").
And("team_repo.org_id = ?", team.OrgID).
And("team_repo.team_id = ?", team.ID)
}
} else {
sess = sess.And("heatmap_contribution.user_id = ?", user.ID)
}
return hdata, sess.
GroupBy(groupByName).
OrderBy("timestamp").
Find(&hdata)
@@ -4,19 +4,43 @@
package activities_test
import (
"crypto/sha1"
"fmt"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/organization"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetUserHeatmapDataByUser(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
// Mock time
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
insertHeatmapContribution(t, 2, 1, "user2-public-author-date", 1603009800)
insertHeatmapContribution(t, 2, 1, "user2-public-same-bucket", 1603009850)
insertHeatmapContribution(t, 2, 2, "user2-private-hidden", 1603010700)
insertHeatmapContribution(t, 2, 4, "user2-hidden-owner-hidden", 1603010700)
insertHeatmapContribution(t, 2, 1, "user2-future-hidden", timeutil.TimeStampNow()+900)
insertHeatmapContribution(t, 2, 999999, "user2-deleted-repo-hidden", 1603011600)
insertHeatmapContribution(t, 3, 1, "other-author-ignored", 1603010700)
setUserVisibility(t, 5, structs.VisibleTypePrivate)
testCases := []struct {
desc string
userID int64
@@ -25,73 +49,211 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
JSONResult string
}{
{
"self looks at action in private repo",
2, 2, 1, `[{"timestamp":1603227600,"contributions":1}]`,
"self sees public author-date contributions only",
2, 2, 2, `[{"timestamp":1603009800,"contributions":2}]`,
},
{
"admin looks at action in private repo",
2, 1, 1, `[{"timestamp":1603227600,"contributions":1}]`,
"admin sees public author-date contributions only",
2, 1, 2, `[{"timestamp":1603009800,"contributions":2}]`,
},
{
"other user looks at action in private repo",
2, 3, 0, `[]`,
"other user sees public author-date contributions only",
2, 3, 2, `[{"timestamp":1603009800,"contributions":2}]`,
},
{
"nobody looks at action in private repo",
2, 0, 0, `[]`,
"anonymous sees public author-date contributions only",
2, 0, 2, `[{"timestamp":1603009800,"contributions":2}]`,
},
{
"collaborator looks at action in private repo",
16, 15, 1, `[{"timestamp":1603267200,"contributions":1}]`,
},
{
"no action action not performed by target user",
"different author is not counted for target user",
3, 3, 0, `[]`,
},
{
"multiple actions performed with two grouped together",
10, 10, 3, `[{"timestamp":1603009800,"contributions":1},{"timestamp":1603010700,"contributions":2}]`,
},
}
// Prepare
assert.NoError(t, unittest.PrepareTestDatabase())
// Mock time
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
var doer *user_model.User
if tc.doerID != 0 {
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
}
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
require.NoError(t, err)
assert.Equal(t, tc.CountResult, countHeatmapContributions(heatmap), "testcase '%s'", tc.desc)
// Test JSON rendering
jsonData, err := json.Marshal(heatmap)
require.NoError(t, err)
assert.JSONEq(t, tc.JSONResult, string(jsonData))
})
}
}
func TestGetUserHeatmapDataByUserUsesCurrentRepoVisibility(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
for _, tc := range testCases {
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
insertHeatmapContribution(t, 2, 1, "user2-visibility-flip", 1603009800)
var doer *user_model.User
if tc.doerID != 0 {
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
}
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assertHeatmapJSON(t, heatmap, `[{
"timestamp": 1603009800,
"contributions": 1
}]`)
// get the action for comparison
actions, count, err := activities_model.GetFeeds(t.Context(), activities_model.GetFeedsOptions{
RequestedUser: user,
Actor: doer,
IncludePrivate: true,
OnlyPerformedBy: true,
IncludeDeleted: true,
})
assert.NoError(t, err)
setRepoPrivate(t, 1, true)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assert.Empty(t, heatmap)
// Get the heatmap and compare
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
var contributions int
for _, hm := range heatmap {
contributions += int(hm.Contributions)
}
assert.NoError(t, err)
assert.Len(t, actions, contributions, "invalid action count: did the test data became too old?")
assert.Equal(t, count, int64(contributions))
assert.Equal(t, tc.CountResult, contributions, "testcase '%s'", tc.desc)
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, true))
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assertHeatmapJSON(t, heatmap, `[{
"timestamp": 1603009800,
"contributions": 1
}]`)
}
// Test JSON rendering
jsonData, err := json.Marshal(heatmap)
assert.NoError(t, err)
assert.JSONEq(t, tc.JSONResult, string(jsonData))
func TestGetUserHeatmapDataByOrgTeam(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 3, false))
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
insertHeatmapContribution(t, 2, 32, "org-team-public", 1603009800)
insertHeatmapContribution(t, 2, 3, "org-team-private-hidden", 1603010700)
insertHeatmapContribution(t, 2, 1, "non-org-repo-ignored", 1603011600)
org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3})
team := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: 7})
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
heatmap, err := activities_model.GetUserHeatmapDataByOrgTeam(t.Context(), org, team, doer)
require.NoError(t, err)
assert.Equal(t, 1, countHeatmapContributions(heatmap))
jsonData, err := json.Marshal(heatmap)
require.NoError(t, err)
assert.JSONEq(t, `[{"timestamp":1603009800,"contributions":1}]`, string(jsonData))
}
func TestUserHeatmapPrivateContributionsOptIn(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
insertHeatmapContribution(t, 16, 21, "user16-public", 1603009800)
insertHeatmapContribution(t, 16, 22, "user16-private-one", 1603009850)
insertHeatmapContribution(t, 16, 22, "user16-private-two", 1603010700)
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 16})
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
authenticated := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3})
collaborator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15})
for _, doer := range []*user_model.User{nil, target, admin, authenticated, collaborator} {
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, doer)
require.NoError(t, err)
assert.Equal(t, 1, countHeatmapContributions(heatmap), "private contributions should be hidden by default from %s", heatmapDoerName(doer))
}
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, true))
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assertHeatmapJSON(t, heatmap, `[{"timestamp":1603009800,"contributions":2},{"timestamp":1603010700,"contributions":1}]`)
target.KeepActivityPrivate = true
_, err = db.GetEngine(t.Context()).ID(target.ID).Cols("keep_activity_private").Update(target)
require.NoError(t, err)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assert.Empty(t, heatmap)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, authenticated)
require.NoError(t, err)
assert.Empty(t, heatmap)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, target)
require.NoError(t, err)
assert.Equal(t, 3, countHeatmapContributions(heatmap))
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, admin)
require.NoError(t, err)
assert.Equal(t, 3, countHeatmapContributions(heatmap))
}
func insertHeatmapContribution(t *testing.T, userID, repoID int64, commitSHA string, authorUnix timeutil.TimeStamp) {
t.Helper()
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: fmt.Sprintf("%040x", sha1.Sum([]byte(commitSHA))),
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
AuthorUnix: authorUnix,
}))
}
func setRepoPrivate(t *testing.T, repoID int64, isPrivate bool) {
t.Helper()
repo := &repo_model.Repository{ID: repoID, IsPrivate: isPrivate}
_, err := db.GetEngine(t.Context()).ID(repoID).Cols("is_private").Update(repo)
require.NoError(t, err)
}
func setUserVisibility(t *testing.T, userID int64, visibility structs.VisibleType) {
t.Helper()
user := &user_model.User{ID: userID, Visibility: visibility}
_, err := db.GetEngine(t.Context()).ID(userID).Cols("visibility").Update(user)
require.NoError(t, err)
}
func assertHeatmapJSON(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected string) {
t.Helper()
jsonData, err := json.Marshal(heatmap)
require.NoError(t, err)
assert.JSONEq(t, expected, string(jsonData))
var decoded []map[string]any
require.NoError(t, json.Unmarshal(jsonData, &decoded))
for _, entry := range decoded {
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, mapKeys(entry))
}
}
func mapKeys[K comparable, V any](m map[K]V) []K {
keys := make([]K, 0, len(m))
for key := range m {
keys = append(keys, key)
}
return keys
}
func countHeatmapContributions(heatmap []*activities_model.UserHeatmapData) int {
var contributions int
for _, hm := range heatmap {
contributions += int(hm.Contributions)
}
return contributions
}
func heatmapDoerName(doer *user_model.User) string {
if doer == nil {
return "anonymous"
}
return doer.Name
}
@@ -405,6 +405,7 @@ func prepareMigrationTasks() []*migration {
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
newMigration(331, "Create heatmap contribution table", v1_26.CreateHeatmapContributionTable),
}
return preparedMigrations
}
@@ -0,0 +1,38 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_26
import (
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/xorm"
"xorm.io/xorm/schemas"
)
type HeatmapContribution struct { //revive:disable-line:exported
ID int64 `xorm:"pk autoincr"`
UserID int64 `xorm:"NOT NULL"`
RepoID int64 `xorm:"NOT NULL"`
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
// TableIndices implements xorm's TableIndices interface.
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
return []*schemas.Index{uniqueContribution, userAuthorRepo}
}
func CreateHeatmapContributionTable(x *xorm.Engine) error {
_, err := x.SyncWithOptions(xorm.SyncOptions{IgnoreDropIndices: true}, new(HeatmapContribution))
return err
}
@@ -7,6 +7,7 @@ import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"code.gitea.io/gitea/models/db"
@@ -19,6 +20,8 @@ import (
"xorm.io/xorm/convert"
)
const SettingsKeyIncludePrivateContributions = "include_private_contributions"
// Setting is a key value store of user settings
type Setting struct {
ID int64 `xorm:"pk autoincr"`
@@ -254,3 +257,17 @@ func SetUserSettingJSON[T any](ctx context.Context, userID int64, key string, va
}
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
}
// GetIncludePrivateContributions returns whether a user opted in to private heatmap contributions.
func GetIncludePrivateContributions(ctx context.Context, userID int64) (bool, error) {
value, err := GetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, "false")
if err != nil {
return false, err
}
return strconv.ParseBool(value)
}
// SetIncludePrivateContributions stores whether a user opted in to private heatmap contributions.
func SetIncludePrivateContributions(ctx context.Context, userID int64, include bool) error {
return SetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, strconv.FormatBool(include))
}
+6 -4
View File
@@ -80,8 +80,9 @@ type UserSettings struct {
Theme string `json:"theme"`
DiffViewStyle string `json:"diff_view_style"`
// Privacy
HideEmail bool `json:"hide_email"`
HideActivity bool `json:"hide_activity"`
HideEmail bool `json:"hide_email"`
HideActivity bool `json:"hide_activity"`
IncludePrivateContributions bool `json:"include_private_contributions"`
}
// UserSettingsOptions represents options to change user settings
@@ -95,8 +96,9 @@ type UserSettingsOptions struct {
Theme *string `json:"theme"`
DiffViewStyle *string `json:"diff_view_style"`
// Privacy
HideEmail *bool `json:"hide_email"`
HideActivity *bool `json:"hide_activity"`
HideEmail *bool `json:"hide_email"`
HideActivity *bool `json:"hide_activity"`
IncludePrivateContributions *bool `json:"include_private_contributions"`
}
// RenameUserOption options when renaming a user
@@ -708,6 +708,8 @@
"settings.privacy": "Privacy",
"settings.keep_activity_private": "Hide Activity from profile page",
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
"settings.include_private_contributions": "Show private contributions on profile heatmap",
"settings.include_private_contributions_popup": "Publicly reveals only contribution dates and counts from non-public repositories, never repository or commit details.",
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
"settings.federated_avatar_lookup": "Federated Avatar Lookup",
"settings.enable_custom_avatar": "Use Custom Avatar",
@@ -24,7 +24,12 @@ func GetUserSettings(ctx *context.APIContext) {
// responses:
// "200":
// "$ref": "#/responses/UserSettings"
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
if err != nil {
ctx.APIErrorInternal(err)
return
}
ctx.JSON(http.StatusOK, settings)
}
// UpdateUserSettings returns user settings
@@ -46,20 +51,26 @@ func UpdateUserSettings(ctx *context.APIContext) {
form := web.GetForm(ctx).(*api.UserSettingsOptions)
opts := &user_service.UpdateOptions{
FullName: optional.FromPtr(form.FullName),
Description: optional.FromPtr(form.Description),
Website: optional.FromPtr(form.Website),
Location: optional.FromPtr(form.Location),
Language: optional.FromPtr(form.Language),
Theme: optional.FromPtr(form.Theme),
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
FullName: optional.FromPtr(form.FullName),
Description: optional.FromPtr(form.Description),
Website: optional.FromPtr(form.Website),
Location: optional.FromPtr(form.Location),
Language: optional.FromPtr(form.Language),
Theme: optional.FromPtr(form.Theme),
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
IncludePrivateContributions: optional.FromPtr(form.IncludePrivateContributions),
}
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
ctx.APIErrorInternal(err)
return
}
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
if err != nil {
ctx.APIErrorInternal(err)
return
}
ctx.JSON(http.StatusOK, settings)
}
@@ -48,16 +48,32 @@ func Profile(ctx *context.Context) {
ctx.Data["PageIsSettingsProfile"] = true
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
if !loadProfilePrivateContributionSetting(ctx) {
return
}
ctx.HTML(http.StatusOK, tplSettingsProfile)
}
func loadProfilePrivateContributionSetting(ctx *context.Context) bool {
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, ctx.Doer.ID)
if err != nil {
ctx.ServerError("GetIncludePrivateContributions", err)
return false
}
ctx.Data["IncludePrivateContributions"] = includePrivateContributions
return true
}
// ProfilePost response for change user's profile
func ProfilePost(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("settings_title")
ctx.Data["PageIsSettingsProfile"] = true
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
if !loadProfilePrivateContributionSetting(ctx) {
return
}
if ctx.HasError() {
ctx.HTML(http.StatusOK, tplSettingsProfile)
@@ -94,12 +110,13 @@ func ProfilePost(ctx *context.Context) {
}
opts := &user_service.UpdateOptions{
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
Description: optional.Some(form.Description),
Website: optional.Some(form.Website),
Location: optional.Some(form.Location),
Visibility: optional.Some(form.Visibility),
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
Description: optional.Some(form.Description),
Website: optional.Some(form.Website),
Location: optional.Some(form.Location),
Visibility: optional.Some(form.Visibility),
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
IncludePrivateContributions: optional.Some(form.IncludePrivateContributions),
}
if form.FullName != "" {
+16 -11
View File
@@ -86,18 +86,23 @@ func toUser(ctx context.Context, user *user_model.User, signed, authed bool) *ap
}
// User2UserSettings return UserSettings based on a user
func User2UserSettings(user *user_model.User) api.UserSettings {
return api.UserSettings{
FullName: user.FullName,
Website: user.Website,
Location: user.Location,
Language: user.Language,
Description: user.Description,
Theme: user.Theme,
HideEmail: user.KeepEmailPrivate,
HideActivity: user.KeepActivityPrivate,
DiffViewStyle: user.DiffViewStyle,
func User2UserSettings(ctx context.Context, user *user_model.User) (api.UserSettings, error) {
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
if err != nil {
return api.UserSettings{}, err
}
return api.UserSettings{
FullName: user.FullName,
Website: user.Website,
Location: user.Location,
Language: user.Language,
Description: user.Description,
Theme: user.Theme,
HideEmail: user.KeepEmailPrivate,
HideActivity: user.KeepActivityPrivate,
IncludePrivateContributions: includePrivateContributions,
DiffViewStyle: user.DiffViewStyle,
}, nil
}
// ToUserAndPermission return User and its collaboration permission for a repository
@@ -208,14 +208,15 @@ func (f *IntrospectTokenForm) Validate(req *http.Request, errs binding.Errors) b
// UpdateProfileForm form for updating profile
type UpdateProfileForm struct {
Name string `binding:"Username;MaxSize(40)"`
FullName string `binding:"MaxSize(100)"`
KeepEmailPrivate bool
Website string `binding:"ValidSiteUrl;MaxSize(255)"`
Location string `binding:"MaxSize(50)"`
Description string `binding:"MaxSize(255)"`
Visibility structs.VisibleType
KeepActivityPrivate bool
Name string `binding:"Username;MaxSize(40)"`
FullName string `binding:"MaxSize(100)"`
KeepEmailPrivate bool
Website string `binding:"ValidSiteUrl;MaxSize(255)"`
Location string `binding:"MaxSize(50)"`
Description string `binding:"MaxSize(255)"`
Visibility structs.VisibleType
KeepActivityPrivate bool
IncludePrivateContributions bool
}
// Validate validates the fields
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
defer gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
indexHeatmap := false
if len(results) > 0 {
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
if result.RefName.IsPull() {
continue
}
if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
indexHeatmap = true
}
// Create reference
if result.OldCommitID == "" {
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
NewCommitID: newCommitID,
}, theCommits)
}
if indexHeatmap {
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
}
}
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
isEmpty, err := gitRepo.IsEmpty()
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
}
notify_service.ChangeDefaultBranch(ctx, repo)
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
}
return nil
}
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
}
}
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
}
return repo, nil
}
@@ -0,0 +1,109 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"context"
"strings"
activities_model "code.gitea.io/gitea/models/activities"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/timeutil"
)
const heatmapIndexCommitsPageSize = 100
// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch.
func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error {
if repo == nil || repo.ID == 0 {
return nil
}
if repo.DefaultBranch == "" || repo.IsEmpty {
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
}
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
if err != nil {
return err
}
defer closer.Close()
head, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
if git.IsErrNotExist(err) {
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
}
if err != nil {
return err
}
currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0)
now := timeutil.TimeStampNow()
for page := 1; ; page++ {
commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "")
if err != nil {
return err
}
if len(commits) == 0 {
break
}
for _, commit := range commits {
commitSHA := commit.ID.String()
if commit.Author == nil || commit.Author.Email == "" {
continue
}
authorUnix := timeutil.TimeStamp(commit.Author.When.Unix())
if authorUnix > now {
continue
}
userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email)
if err != nil {
return err
}
if !ok {
continue
}
if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repo.ID,
CommitSHA: commitSHA,
AuthorEmail: strings.ToLower(commit.Author.Email),
AuthorUnix: authorUnix,
}); err != nil {
return err
}
currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{
RepoID: repo.ID,
CommitSHA: commitSHA,
UserID: userID,
})
}
if len(commits) < heatmapIndexCommitsPageSize {
break
}
}
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities)
}
func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) {
address, err := user_model.GetEmailAddressByEmail(ctx, email)
if user_model.IsErrEmailAddressNotExist(err) {
return 0, false, nil
}
if err != nil {
return 0, false, err
}
if !address.IsActivated {
return 0, false, nil
}
return address.UID, true, nil
}
@@ -0,0 +1,218 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"fmt"
"strings"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{
{Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
{Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
{Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, int64(2), contributions[0].UserID)
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail)
assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix)
emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com")
require.NoError(t, err)
emailAddress.UID = 1
require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress))
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
contributions = loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, int64(1), contributions[0].UserID)
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
}
func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{
{Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
{Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
{Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 2)
assert.Equal(t, int64(2), contributions[0].UserID)
assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA)
assert.Equal(t, int64(1), contributions[1].UserID)
assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA)
for _, contribution := range contributions {
assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA)
}
deleteMainRef(t, repo)
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
}
func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
})
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
{
RefFullName: git.RefNameFromBranch("main"),
OldCommitID: commits["initial"],
NewCommitID: newCommits["pushed"],
PusherID: 1,
RepoUserName: repo.OwnerName,
RepoName: repo.Name,
},
}))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 2)
assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
})
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
{
RefFullName: git.RefNameFromBranch("feature"),
OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(),
NewCommitID: featureCommits["feature-pushed"],
PusherID: 1,
RepoUserName: repo.OwnerName,
RepoName: repo.Name,
},
}))
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
}
type heatmapIndexTestCommit struct {
Branch string
Mark string
Parent string
AuthorName string
AuthorEmail string
CommitterName string
CommitterEmail string
AuthorDate string
}
func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) {
t.Helper()
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
require.NoError(t, err)
marks := runFastImport(t, repo, commits)
repo.IsEmpty = false
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
return repo, marks
}
func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string {
t.Helper()
var stream strings.Builder
branchTips := make(map[string]string)
for i, commit := range commits {
mark := fmt.Sprintf(":%d", i+1)
branchRef := "refs/heads/" + commit.Branch
stream.WriteString("commit " + branchRef + "\n")
stream.WriteString("mark " + mark + "\n")
stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
message := "heatmap " + commit.Mark
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
if parentMark := branchTips[commit.Branch]; parentMark != "" {
stream.WriteString("from " + parentMark + "\n")
} else if commit.Parent != "" {
stream.WriteString("from " + commit.Parent + "\n")
}
content := commit.Mark + "\n"
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
branchTips[commit.Branch] = mark
}
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
WithDir(repo.RepoPath()).
WithStdinCopy(strings.NewReader(stream.String())).
Run(t.Context()))
commitSHAs := make(map[string]string, len(commits))
for _, commit := range commits {
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
AddOptionFormat("--grep=%s", "heatmap "+commit.Mark).
AddDynamicArguments("refs/heads/" + commit.Branch).
WithDir(repo.RepoPath()).
RunStdString(t.Context())
require.NoError(t, err)
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
}
return commitSHAs
}
func signatureLine(kind, name, email, date string) string {
parsed, err := time.Parse(time.RFC3339, date)
if err != nil {
panic(err)
}
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
}
func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
t.Helper()
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
require.NoError(t, err)
return contributions
}
func deleteMainRef(t *testing.T, repo *repo_model.Repository) {
t.Helper()
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
}
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
}
}
return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
if opts.Mirror {
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
if err != nil {
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
}
return repo, nil
})
if err != nil {
return nil, err
}
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
}
return repo, nil
}
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
}
}
indexHeatmap := false
if !opts.IsDelRef() {
branch := opts.RefFullName.BranchName()
@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
log.Error("DelRepoDivergenceFromCache: %v", err)
}
indexHeatmap = true
} else {
if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
log.Error("DelDivergenceFromCache: %v", err)
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
pushDeleteBranch(ctx, repo, pusher, opts)
}
if indexHeatmap {
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
}
}
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
+19 -1
View File
@@ -8,6 +8,7 @@ import (
"fmt"
auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
password_module "code.gitea.io/gitea/modules/auth/password"
"code.gitea.io/gitea/modules/optional"
@@ -47,6 +48,7 @@ type UpdateOptions struct {
IsRestricted optional.Option[bool]
Visibility optional.Option[structs.VisibleType]
KeepActivityPrivate optional.Option[bool]
IncludePrivateContributions optional.Option[bool]
Language optional.Option[string]
Theme optional.Option[string]
DiffViewStyle optional.Option[string]
@@ -182,7 +184,23 @@ func UpdateUser(ctx context.Context, u *user_model.User, opts *UpdateOptions) er
cols = append(cols, "last_login_unix")
}
return user_model.UpdateUserCols(ctx, u, cols...)
if len(cols) > 0 || opts.IncludePrivateContributions.Has() {
return db.WithTx(ctx, func(ctx context.Context) error {
if len(cols) > 0 {
if err := user_model.UpdateUserCols(ctx, u, cols...); err != nil {
return err
}
}
if opts.IncludePrivateContributions.Has() {
return user_model.SetIncludePrivateContributions(ctx, u.ID, opts.IncludePrivateContributions.Value())
}
return nil
})
}
return nil
}
type UpdateAuthOptions struct {
@@ -44,6 +44,7 @@ func TestUpdateUser(t *testing.T) {
IsAdmin: UpdateOptionFieldFromValue(true),
Visibility: optional.Some(structs.VisibleTypePrivate),
KeepActivityPrivate: optional.Some(true),
IncludePrivateContributions: optional.Some(true),
Language: optional.Some("lang"),
Theme: optional.Some("theme"),
DiffViewStyle: optional.Some("split"),
@@ -66,6 +67,9 @@ func TestUpdateUser(t *testing.T) {
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
assert.NoError(t, err)
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
assert.Equal(t, opts.Language.Value(), user.Language)
assert.Equal(t, opts.Theme.Value(), user.Theme)
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
@@ -86,6 +90,9 @@ func TestUpdateUser(t *testing.T) {
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
includePrivateContributions, err = user_model.GetIncludePrivateContributions(t.Context(), user.ID)
assert.NoError(t, err)
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
assert.Equal(t, opts.Language.Value(), user.Language)
assert.Equal(t, opts.Theme.Value(), user.Theme)
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
+9 -1
View File
@@ -29655,6 +29655,10 @@
"type": "boolean",
"x-go-name": "HideEmail"
},
"include_private_contributions": {
"type": "boolean",
"x-go-name": "IncludePrivateContributions"
},
"language": {
"type": "string",
"x-go-name": "Language"
@@ -29699,6 +29703,10 @@
"type": "boolean",
"x-go-name": "HideEmail"
},
"include_private_contributions": {
"type": "boolean",
"x-go-name": "IncludePrivateContributions"
},
"language": {
"type": "string",
"x-go-name": "Language"
@@ -31073,4 +31081,4 @@
"TOTPHeader": []
}
]
}
}
@@ -88,6 +88,13 @@
</div>
</div>
<div class="field">
<div class="ui checkbox" id="include-private-contributions">
<label data-tooltip-content="{{ctx.Locale.Tr "settings.include_private_contributions_popup"}}"><strong>{{ctx.Locale.Tr "settings.include_private_contributions"}}</strong></label>
<input name="include_private_contributions" type="checkbox" {{if .IncludePrivateContributions}}checked{{end}}>
</div>
</div>
<div class="divider"></div>
<div class="field">
@@ -4,17 +4,19 @@
package integration
import (
"fmt"
"net/http"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestUserHeatmap(t *testing.T) {
@@ -23,17 +25,23 @@ func TestUserHeatmap(t *testing.T) {
normalUsername := "user2"
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
fakeNow := time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local)
fakeNow := time.Date(2011, 10, 21, 0, 0, 0, 0, time.Local)
timeutil.MockSet(fakeNow)
defer timeutil.MockUnset()
req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)).
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-author-date", 1319068800)
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-same-bucket", 1319068850)
testHeatmapSeedContribution(t, 2, 2, "api-user2-private-hidden", 1319070600)
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", normalUsername).
AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var heatmap []*activities_model.UserHeatmapData
DecodeJSON(t, resp, &heatmap)
var dummyheatmap []*activities_model.UserHeatmapData
dummyheatmap = append(dummyheatmap, &activities_model.UserHeatmapData{Timestamp: 1603227600, Contributions: 1})
assert.Equal(t, dummyheatmap, heatmap)
assert.Equal(t, []*activities_model.UserHeatmapData{
{Timestamp: 1319068800, Contributions: 2},
}, heatmap)
}
@@ -4,22 +4,40 @@
package integration
import (
"crypto/sha1"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type testHeatmapWebResponse struct {
HeatmapData [][2]int64 `json:"heatmapData"`
TotalContributions int64 `json:"totalContributions"`
}
func TestHeatmapEndpoints(t *testing.T) {
defer tests.PrepareTestEnv(t)()
// Mock time so fixture actions fall within the heatmap's time window
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-one", 1603009800)
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-two", 1603009850)
testHeatmapSeedContribution(t, 2, 2, "web-user2-private-hidden", 1603010700)
session := loginUser(t, "user2")
@@ -28,11 +46,15 @@ func TestHeatmapEndpoints(t *testing.T) {
req := NewRequest(t, "GET", "/user2/-/heatmap")
resp := session.MakeRequest(t, req, http.StatusOK)
var result map[string]any
DecodeJSON(t, resp, &result)
assert.Contains(t, result, "heatmapData")
assert.Contains(t, result, "totalContributions")
assert.Positive(t, result["totalContributions"])
webHeatmap := testHeatmapDecodeWebResponse(t, resp)
req = NewRequest(t, "GET", "/api/v1/users/user2/heatmap")
resp = session.MakeRequest(t, req, http.StatusOK)
var apiHeatmap []*activities_model.UserHeatmapData
DecodeJSON(t, resp, &apiHeatmap)
assert.Equal(t, testHeatmapSumAPIContributions(apiHeatmap), webHeatmap.TotalContributions)
assert.Equal(t, int64(2), webHeatmap.TotalContributions)
})
t.Run("OrgDashboard", func(t *testing.T) {
@@ -57,3 +79,71 @@ func TestHeatmapEndpoints(t *testing.T) {
assert.Contains(t, result, "totalContributions")
})
}
func testHeatmapSeedContribution(t *testing.T, userID, repoID int64, label string, authorUnix timeutil.TimeStamp) string {
t.Helper()
commitSHA := fmt.Sprintf("%040x", sha1.Sum([]byte(label)))
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
AuthorUnix: authorUnix,
}))
return commitSHA
}
func testHeatmapDecodeWebResponse(t *testing.T, resp *httptest.ResponseRecorder) testHeatmapWebResponse {
t.Helper()
var result testHeatmapWebResponse
DecodeJSON(t, resp, &result)
return result
}
func testHeatmapSumAPIContributions(heatmap []*activities_model.UserHeatmapData) int64 {
var total int64
for _, item := range heatmap {
total += item.Contributions
}
return total
}
func testHeatmapAssertAggregateOnlyAPIResponse(t *testing.T, body []byte) {
t.Helper()
var decoded []map[string]any
require.NoError(t, json.Unmarshal(body, &decoded))
for _, entry := range decoded {
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, testHeatmapMapKeys(entry))
}
}
func testHeatmapAssertAggregateOnlyWebResponse(t *testing.T, body []byte) {
t.Helper()
var decoded map[string]any
require.NoError(t, json.Unmarshal(body, &decoded))
assert.ElementsMatch(t, []string{"heatmapData", "totalContributions"}, testHeatmapMapKeys(decoded))
}
func testHeatmapAssertNoPrivateMetadata(t *testing.T, body []byte, forbidden ...string) {
t.Helper()
response := string(body)
for _, value := range forbidden {
assert.NotContains(t, response, value)
}
for _, value := range []string{"repo_id", "repository_id", "repoID", "commit_sha", "commit_message", "branch", "url", "action_id"} {
assert.NotContains(t, response, value)
}
}
func testHeatmapMapKeys[K comparable, V any](m map[K]V) []K {
keys := make([]K, 0, len(m))
for key := range m {
keys = append(keys, key)
}
return keys
}
@@ -7,16 +7,20 @@ import (
"fmt"
"net/http"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
@@ -24,6 +28,8 @@ const (
privateActivityTestUser = "user2"
)
const privateActivityHeatmapOptInUser = "user16"
// org3 is an organization so it is not usable here
const privateActivityTestOtherUser = "user4"
@@ -41,6 +47,7 @@ func testPrivateActivityDoSomethingForActionEntries(t *testing.T) {
Title: "test",
}).AddTokenAuth(token)
session.MakeRequest(t, req, http.StatusCreated)
testHeatmapSeedContribution(t, 2, 1, "private-activity-action-public", timeutil.TimeStampNow()-900)
}
// private activity helpers
@@ -56,6 +63,26 @@ func testPrivateActivityHelperEnablePrivateActivity(t *testing.T) {
session.MakeRequest(t, req, http.StatusSeeOther)
}
func testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t *testing.T, session *TestSession, username string, include bool) {
values := map[string]string{
"name": username,
"email": username + "@example.com",
"language": "en-US",
}
if include {
values["include_private_contributions"] = "1"
}
req := NewRequestWithValues(t, "POST", "/user/settings", values)
session.MakeRequest(t, req, http.StatusSeeOther)
}
func testPrivateActivityHelperAssertIncludePrivateContributions(t *testing.T, username string, expected bool) {
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: username})
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
require.NoError(t, err)
assert.Equal(t, expected, includePrivateContributions)
}
func testPrivateActivityHelperHasVisibleActivitiesInHTMLDoc(htmlDoc *HTMLDoc) bool {
return htmlDoc.doc.Find("#activity-feed").Find(".flex-item").Length() > 0
}
@@ -136,6 +163,196 @@ func testPrivateActivityHelperHasHeatmapContentFromSession(t *testing.T, session
return len(items) != 0
}
func testPrivateActivityHelperGetAPIHeatmapFromPublic(t *testing.T) ([]*activities_model.UserHeatmapData, []byte) {
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", privateActivityHeatmapOptInUser)
resp := MakeRequest(t, req, http.StatusOK)
body := resp.Body.Bytes()
var items []*activities_model.UserHeatmapData
DecodeJSON(t, resp, &items)
return items, body
}
func testPrivateActivityHelperGetAPIHeatmapFromSession(t *testing.T, session *TestSession) ([]*activities_model.UserHeatmapData, []byte) {
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeReadUser)
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", privateActivityHeatmapOptInUser).
AddTokenAuth(token)
resp := session.MakeRequest(t, req, http.StatusOK)
body := resp.Body.Bytes()
var items []*activities_model.UserHeatmapData
DecodeJSON(t, resp, &items)
return items, body
}
func testPrivateActivityHelperGetWebHeatmapFromPublic(t *testing.T) (testHeatmapWebResponse, []byte) {
req := NewRequestf(t, "GET", "/%s/-/heatmap", privateActivityHeatmapOptInUser)
resp := MakeRequest(t, req, http.StatusOK)
body := resp.Body.Bytes()
return testHeatmapDecodeWebResponse(t, resp), body
}
func testPrivateActivityHelperSeedHeatmapContributions(t *testing.T) string {
t.Helper()
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
testHeatmapSeedContribution(t, 16, 21, "private-activity-public", 1603009800)
privateSHA := testHeatmapSeedContribution(t, 16, 22, "private-activity-private-one", 1603009850)
testHeatmapSeedContribution(t, 16, 22, "private-activity-private-two", 1603010700)
return privateSHA
}
func testPrivateActivityHelperAssertAPIHeatmapTotal(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected int64) {
t.Helper()
assert.Equal(t, expected, testHeatmapSumAPIContributions(heatmap))
}
// check private contribution opt-in settings persistence and ownership
func TestPrivateActivityIncludePrivateContributionsWebSettingsPersistence(t *testing.T) {
defer tests.PrepareTestEnv(t)()
session := loginUser(t, privateActivityTestUser)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, session, privateActivityTestUser, true)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, true)
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: privateActivityTestUser})
assert.False(t, user.KeepActivityPrivate, "private contribution opt-in must not hide the whole activity heatmap")
req := NewRequest(t, "GET", "/user/settings")
resp := session.MakeRequest(t, req, http.StatusOK)
htmlDoc := NewHTMLParser(t, resp.Body)
assert.Equal(t, 1, htmlDoc.doc.Find("#include-private-contributions input[name='include_private_contributions']:checked").Length())
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, session, privateActivityTestUser, false)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
}
func TestPrivateActivityIncludePrivateContributionsAPISettingsPersistence(t *testing.T) {
defer tests.PrepareTestEnv(t)()
session := loginUser(t, privateActivityTestUser)
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeReadUser, auth_model.AccessTokenScopeWriteUser)
includePrivateContributions := true
req := NewRequestWithJSON(t, "PATCH", "/api/v1/user/settings", &api.UserSettingsOptions{
IncludePrivateContributions: &includePrivateContributions,
}).AddTokenAuth(token)
resp := session.MakeRequest(t, req, http.StatusOK)
settings := DecodeJSON(t, resp, &api.UserSettings{})
assert.True(t, settings.IncludePrivateContributions)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, true)
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: privateActivityTestUser})
assert.False(t, user.KeepActivityPrivate, "API opt-in must not set KeepActivityPrivate")
req = NewRequest(t, "GET", "/api/v1/user/settings").AddTokenAuth(token)
resp = session.MakeRequest(t, req, http.StatusOK)
settings = DecodeJSON(t, resp, &api.UserSettings{})
assert.True(t, settings.IncludePrivateContributions)
includePrivateContributions = false
req = NewRequestWithJSON(t, "PATCH", "/api/v1/user/settings", &api.UserSettingsOptions{
IncludePrivateContributions: &includePrivateContributions,
}).AddTokenAuth(token)
resp = session.MakeRequest(t, req, http.StatusOK)
settings = DecodeJSON(t, resp, &api.UserSettings{})
assert.False(t, settings.IncludePrivateContributions)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
}
func TestPrivateActivityPrivateContributionsHiddenByDefault(t *testing.T) {
defer tests.PrepareTestEnv(t)()
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
testPrivateActivityHelperSeedHeatmapContributions(t)
publicHeatmap, _ := testPrivateActivityHelperGetAPIHeatmapFromPublic(t)
testPrivateActivityHelperAssertAPIHeatmapTotal(t, publicHeatmap, 1)
for _, testCase := range []struct {
name string
username string
}{
{"owner", privateActivityHeatmapOptInUser},
{"admin", privateActivityTestAdmin},
{"collaborator", "user15"},
{"unrelated", privateActivityTestOtherUser},
} {
t.Run(testCase.name, func(t *testing.T) {
session := loginUser(t, testCase.username)
heatmap, _ := testPrivateActivityHelperGetAPIHeatmapFromSession(t, session)
testPrivateActivityHelperAssertAPIHeatmapTotal(t, heatmap, 1)
})
}
}
func TestPrivateActivityIncludePrivateContributionsEndpointAggregatesOnly(t *testing.T) {
defer tests.PrepareTestEnv(t)()
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
privateSHA := testPrivateActivityHelperSeedHeatmapContributions(t)
ownerSession := loginUser(t, privateActivityHeatmapOptInUser)
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, ownerSession, privateActivityHeatmapOptInUser, true)
apiHeatmap, apiBody := testPrivateActivityHelperGetAPIHeatmapFromPublic(t)
testPrivateActivityHelperAssertAPIHeatmapTotal(t, apiHeatmap, 3)
testHeatmapAssertAggregateOnlyAPIResponse(t, apiBody)
testHeatmapAssertNoPrivateMetadata(t, apiBody,
"big_test_private_3",
"22",
privateSHA,
"private-activity-private-one",
"master",
"/user16/big_test_private_3",
)
webHeatmap, webBody := testPrivateActivityHelperGetWebHeatmapFromPublic(t)
assert.Equal(t, int64(3), webHeatmap.TotalContributions)
testHeatmapAssertAggregateOnlyWebResponse(t, webBody)
testHeatmapAssertNoPrivateMetadata(t, webBody,
"big_test_private_3",
"22",
privateSHA,
"private-activity-private-one",
"master",
"/user16/big_test_private_3",
)
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, ownerSession, privateActivityHeatmapOptInUser, false)
apiHeatmap, _ = testPrivateActivityHelperGetAPIHeatmapFromPublic(t)
testPrivateActivityHelperAssertAPIHeatmapTotal(t, apiHeatmap, 1)
}
func TestPrivateActivityIncludePrivateContributionsWebSettingsAuthBoundary(t *testing.T) {
defer tests.PrepareTestEnv(t)()
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
otherSession := loginUser(t, privateActivityTestOtherUser)
testPrivateActivityHelperSetIncludePrivateContributionsViaWeb(t, otherSession, privateActivityTestOtherUser, true)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestOtherUser, true)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
req := NewRequestWithValues(t, "POST", "/user/settings", map[string]string{
"name": privateActivityTestUser,
"email": privateActivityTestUser + "@example.com",
"language": "en-US",
"include_private_contributions": "1",
})
MakeRequest(t, req, http.StatusSeeOther)
testPrivateActivityHelperAssertIncludePrivateContributions(t, privateActivityTestUser, false)
}
// check activity visibility if the visibility is enabled
func TestPrivateActivityNoVisibleForPublic(t *testing.T) {
+2 -2
View File
@@ -1,4 +1,4 @@
{ dash, hectic, sqlite, postgresql_17, gawk, self }:
{ dash, hectic, sqlite, postgresql_17, gawk, coreutils, self }:
let
shell = "${dash}/bin/dash";
bashOptions = [
@@ -11,7 +11,7 @@ let
migrator = hectic.writeShellApplication {
inherit shell bashOptions;
name = "migrator";
runtimeInputs = [ sqlite postgresql_17 gawk ];
runtimeInputs = [ sqlite postgresql_17 gawk coreutils ];
text = ''
${builtins.readFile hectic.helpers.posix-shell.log}
+31 -13
View File
@@ -154,11 +154,11 @@ init() {
shift 2
;;
--*|-*)
printf 'init argument %s does not exists' "$1"
log error "init argument ${WHITE}$1${NC} does not exists"
exit 9
;;
*)
printf 'init command %s does not exists' "$1"
log error "init subcommand ${WHITE}$1${NC} does not exists"
exit 9
;;
esac
@@ -434,6 +434,8 @@ migrate_down() {
target_migration=$(printf '%s' "$fs_migrations" | sed -n "${target_line}p")
printf '%s' "$target_migration"
fi
return 0
}
migrate_up() {
@@ -488,6 +490,7 @@ migrate_up() {
fi
printf '%s' "$target_migration"
return 0
}
migrate_to() {
@@ -518,6 +521,8 @@ migrate_to() {
printf '%s' "$migration_name"
;;
esac
return 0
}
migration_list() {
@@ -566,7 +571,10 @@ index_of() {
# no subshell, no pipeline
while IFS= read -r m; do
[ "$m" = "$name" ] && { printf '%s\n' "$i"; return 0; }
if [ "$m" = "$name" ]; then
printf '%s\n' "$i"
return 0
fi
i=$((i+1))
done <<EOF
$list
@@ -633,7 +641,8 @@ migrate() {
db_mig_count=$(printf '%s' "$db_migrations" | wc -l)
log debug "mig count: $db_mig_count"
# Log migration lists for debugging
# These counts are useful even at normal verbosity because they summarize the
# migration state before any validation or apply/revert work begins.
fs_mig_count=$(printf '%s' "$fs_migrations" | wc -l)
log info "Filesystem migrations found: ${WHITE}$fs_mig_count"
log info "Database migrations applied: ${WHITE}$db_mig_count"
@@ -696,26 +705,35 @@ migrate() {
log info "Migration history validation: ${GREEN}OK${NC} (${WHITE}$i${NC} migrations match)"
eval "set -- $MIGRATOR_REMAINING_ARS"
target_migration="$("migrate_$MIGRATE_SUBCOMMAND" "$@")"
if ! target_migration="$("migrate_$MIGRATE_SUBCOMMAND" "$@")"; then
log error "failed to resolve migration target for ${WHITE}migrate $MIGRATE_SUBCOMMAND${NC}"
exit 1
fi
if [ -z "$db_migrations" ]; then
log info "it'll firs migration"
log info "starting from clean migration state"
current_idx=0
else
current_migration=$(printf '%s\n' "$db_migrations" | tail -n1)
current_idx=$(index_of "$fs_migrations" "$current_migration")
if ! current_idx=$(index_of "$fs_migrations" "$current_migration"); then
log error "current database migration is missing from filesystem: ${WHITE}$current_migration${NC}"
exit 2
fi
fi
log debug "[$WHITE$fs_migrations$NC]"
log debug "$target_migration"
log debug "filesystem migrations: ${WHITE}$fs_migrations${NC}"
log debug "requested target migration: ${WHITE}${target_migration:-<clean state>}${NC}"
if [ -z "$target_migration" ]; then
target_idx=0
else
target_idx=$(index_of "$fs_migrations" "$target_migration")
if ! target_idx=$(index_of "$fs_migrations" "$target_migration"); then
log error "target migration is missing from filesystem: ${WHITE}$target_migration${NC}"
exit 2
fi
fi
log debug "indexes $WHITE$current_idx$NC $WHITE${target_idx}"
log debug "migration indexes: current=${WHITE}$current_idx${NC} target=${WHITE}$target_idx${NC}"
if [ "$target_idx" -eq "$current_idx" ]; then
if [ "$target_idx" -eq 0 ]; then
@@ -903,11 +921,11 @@ list() {
shift
;;
--*|-*)
log error "init argument $1 does not exists"
log error "list argument ${WHITE}$1${NC} does not exists"
exit 9
;;
*)
log error "init subcommand $1 does not exists"
log error "list subcommand ${WHITE}$1${NC} does not exists"
exit 9
;;
esac
+31
View File
@@ -0,0 +1,31 @@
{
buildGoModule,
fetchFromGitHub,
lib,
}:
buildGoModule {
pname = "plgo";
version = "0.2.1-unstable-2026-04-04";
src = fetchFromGitHub {
owner = "mgr43";
repo = "plgo";
rev = "492abd81f50f5f588794bd3f9fa103d14546b275";
hash = "sha256-Ifd2IFA62VtkqCUP0/cD3jT4SqkjqwCpHVolslQbwkA=";
};
vendorHash = "sha256-KIVHepJRfVa0bkTF1A4g6ZFGxDy8OPzY3iwV3MhRyVA=";
proxyVendor = true;
subPackages = [ "cmd/plgo" ];
doCheck = false;
meta = {
description = "Write PostgreSQL extensions in Go";
homepage = "https://github.com/mgr43/plgo";
license = lib.licenses.bsd3;
mainProgram = "plgo";
};
}
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
"""Local HTTP API for Diffusers Stable Video Diffusion image-to-video."""
import os
import threading
import uuid
from importlib import import_module
from pathlib import Path
from typing import Any
torch = import_module("torch")
uvicorn = import_module("uvicorn")
diffusers = import_module("diffusers")
diffusers_utils = import_module("diffusers.utils")
fastapi = import_module("fastapi")
pil_image = import_module("PIL.Image")
pil_unidentified_image_error = import_module("PIL").UnidentifiedImageError
FastAPI = fastapi.FastAPI
File = fastapi.File
Form = fastapi.Form
HTTPException = fastapi.HTTPException
Request = fastapi.Request
UploadFile = fastapi.UploadFile
StableVideoDiffusionPipeline = diffusers.StableVideoDiffusionPipeline
export_to_video = diffusers_utils.export_to_video
HOST = os.environ.get("SVD_API_HOST", "127.0.0.1")
PORT = int(os.environ.get("SVD_API_PORT", "8000"))
MODEL_ID = os.environ.get("SVD_MODEL_ID", "stabilityai/stable-video-diffusion-img2vid-xt")
CACHE_DIR = os.environ.get("SVD_CACHE_DIR") or os.environ.get("HF_HOME")
OUTPUT_DIR = Path(os.environ.get("SVD_OUTPUT_DIR", "/var/lib/stable-video-diffusion-api/outputs"))
DEVICE = os.environ.get("SVD_DEVICE", "cuda" if torch.cuda.is_available() else "cpu")
DTYPE = os.environ.get("SVD_DTYPE", "float16")
ENABLE_CPU_OFFLOAD = os.environ.get("SVD_ENABLE_CPU_OFFLOAD", "0").lower() in ("1", "true", "yes", "on")
DEFAULT_DECODE_CHUNK_SIZE = int(os.environ.get("SVD_DECODE_CHUNK_SIZE", "8"))
DEFAULT_FPS = int(os.environ.get("SVD_FPS", "7"))
app = FastAPI(title="Stable Video Diffusion API")
pipeline_lock = threading.Lock()
pipeline: Any | None = None
def torch_dtype() -> Any:
dtypes = {
"float16": torch.float16,
"fp16": torch.float16,
"float32": torch.float32,
"fp32": torch.float32,
"bfloat16": torch.bfloat16,
"bf16": torch.bfloat16,
}
try:
return dtypes[DTYPE.lower()]
except KeyError as exc:
raise RuntimeError(f"Unsupported SVD_DTYPE: {DTYPE}") from exc
def get_pipeline() -> Any:
global pipeline
if pipeline is not None:
return pipeline
with pipeline_lock:
if pipeline is not None:
return pipeline
kwargs: dict[str, Any] = {"torch_dtype": torch_dtype()}
if CACHE_DIR:
kwargs["cache_dir"] = CACHE_DIR
loaded = StableVideoDiffusionPipeline.from_pretrained(MODEL_ID, **kwargs)
if ENABLE_CPU_OFFLOAD:
loaded.enable_model_cpu_offload()
else:
loaded.to(DEVICE)
pipeline = loaded
return loaded
def positive_int(name: str, value: Any, default: int) -> int:
if value in (None, ""):
return default
try:
parsed = int(value)
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"{name} must be an integer") from exc
if parsed <= 0:
raise HTTPException(status_code=400, detail=f"{name} must be greater than zero")
return parsed
def optional_int(name: str, value: Any) -> int | None:
if value in (None, ""):
return None
try:
return int(value)
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"{name} must be an integer") from exc
def optional_float(name: str, value: Any) -> float | None:
if value in (None, ""):
return None
try:
return float(value)
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"{name} must be a number") from exc
def load_image_from_path(input_image_path: str) -> Any:
path = Path(input_image_path).expanduser()
if not path.exists() or not path.is_file():
raise HTTPException(status_code=400, detail="input_image_path does not exist or is not a file")
try:
return pil_image.open(path).convert("RGB")
except (OSError, pil_unidentified_image_error) as exc:
raise HTTPException(status_code=400, detail="input_image_path could not be decoded as an image") from exc
async def load_uploaded_image(image: UploadFile) -> Any:
try:
return pil_image.open(image.file).convert("RGB")
except (OSError, pil_unidentified_image_error) as exc:
raise HTTPException(status_code=400, detail="uploaded image could not be decoded") from exc
async def json_payload(request: Request) -> dict[str, Any]:
content_type = request.headers.get("content-type", "")
if not content_type.startswith("application/json"):
return {}
try:
payload = await request.json()
except ValueError as exc:
raise HTTPException(status_code=400, detail="request body must be valid JSON") from exc
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="JSON request body must be an object")
return payload
@app.get("/health")
def health() -> dict[str, Any]:
return {
"status": "ok",
"model_loaded": pipeline is not None,
"model_id": MODEL_ID,
"device": DEVICE,
"dtype": DTYPE,
"cpu_offload": ENABLE_CPU_OFFLOAD,
"cache_dir": CACHE_DIR,
"output_dir": str(OUTPUT_DIR),
}
@app.post("/generate")
async def generate(
request: Request,
image: UploadFile | None = File(default=None),
input_image_path: str | None = Form(default=None),
num_frames: int | None = Form(default=None),
num_inference_steps: int | None = Form(default=None),
fps: int | None = Form(default=None),
decode_chunk_size: int | None = Form(default=None),
seed: int | None = Form(default=None),
motion_bucket_id: int | None = Form(default=None),
noise_aug_strength: float | None = Form(default=None),
) -> dict[str, Any]:
payload = await json_payload(request)
path = input_image_path or payload.get("input_image_path")
upload = image
if upload is None and not path:
raise HTTPException(status_code=400, detail="provide input_image_path or upload image")
if upload is not None and path:
raise HTTPException(status_code=400, detail="provide only one image source")
source_image = await load_uploaded_image(upload) if upload is not None else load_image_from_path(str(path))
chunk_size = positive_int("decode_chunk_size", decode_chunk_size if decode_chunk_size is not None else payload.get("decode_chunk_size"), DEFAULT_DECODE_CHUNK_SIZE)
output_fps = positive_int("fps", fps if fps is not None else payload.get("fps"), DEFAULT_FPS)
request_seed = optional_int("seed", seed if seed is not None else payload.get("seed"))
generator = None
if request_seed is not None:
generator = torch.Generator(device="cpu").manual_seed(request_seed)
call_args: dict[str, Any] = {
"image": source_image,
"decode_chunk_size": chunk_size,
}
for key, value in {
"num_frames": num_frames if num_frames is not None else payload.get("num_frames"),
"num_inference_steps": num_inference_steps if num_inference_steps is not None else payload.get("num_inference_steps"),
"motion_bucket_id": motion_bucket_id if motion_bucket_id is not None else payload.get("motion_bucket_id"),
}.items():
parsed = optional_int(key, value)
if parsed is not None:
call_args[key] = parsed
parsed_noise = optional_float("noise_aug_strength", noise_aug_strength if noise_aug_strength is not None else payload.get("noise_aug_strength"))
if parsed_noise is not None:
call_args["noise_aug_strength"] = parsed_noise
if generator is not None:
call_args["generator"] = generator
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
output_path = OUTPUT_DIR / f"svd-{uuid.uuid4().hex}.mp4"
try:
frames = get_pipeline()(**call_args).frames[0]
export_to_video(frames, str(output_path), fps=output_fps)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {
"output_path": str(output_path),
"model_id": MODEL_ID,
"seed": request_seed,
"fps": output_fps,
}
def main() -> None:
uvicorn.run("app:app", host=HOST, port=PORT)
if __name__ == "__main__":
main()
@@ -0,0 +1,41 @@
{ pkgs }:
let
pythonEnv = pkgs.python3.withPackages (ps: let
torch = ps.torchWithCuda;
accelerate = ps.accelerate.override { inherit torch; };
in [
accelerate
ps.diffusers
ps.fastapi
ps.imageio
ps.imageio-ffmpeg
ps.pillow
ps.python-multipart
ps.safetensors
torch
ps.transformers
ps.uvicorn
]);
in
pkgs.stdenv.mkDerivation {
pname = "stable-video-diffusion-api";
version = "0.1.0";
src = ./.;
nativeBuildInputs = [ pkgs.makeWrapper ];
installPhase = ''
mkdir -p $out/bin $out/libexec/stable-video-diffusion-api
cp $src/app.py $out/libexec/stable-video-diffusion-api/app.py
chmod +x $out/libexec/stable-video-diffusion-api/app.py
makeWrapper ${pythonEnv}/bin/python3 $out/bin/stable-video-diffusion-api \
--add-flags $out/libexec/stable-video-diffusion-api/app.py \
--set-default SVD_API_HOST 127.0.0.1 \
--set-default SVD_API_PORT 8000 \
--set-default SVD_MODEL_ID stabilityai/stable-video-diffusion-img2vid-xt \
--set-default SVD_OUTPUT_DIR /var/lib/stable-video-diffusion-api/outputs
'';
}
+1939
View File
File diff suppressed because it is too large Load Diff
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "which-country-rs"
version = "1.0.0"
edition = "2024"
description = "CLI tool that detects your country by IP and renders an ASCII map"
license = "MIT"
repository = "https://github.com/krisfur/which-country-rs"
keywords = ["cli", "geolocation", "ascii", "map"]
categories = ["command-line-utilities"]
[features]
default = ["geoip"]
geoip = ["reqwest"]
[dependencies]
clap = { version = "4", features = ["derive"] }
reqwest = { version = "0.13", features = ["blocking", "json"], optional = true }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Krzysztof Furman
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+83
View File
@@ -0,0 +1,83 @@
# which-country-rs
A CLI tool that detects your country from your IP address and renders an ASCII map zoomed into it, showing neighbouring borders and country codes.
## Example output
```bash
which-country-rs -c DE
```
```
You appear to be in: Germany (DE)
·········· ·· ······ · ·
· NO · ·· · ···
· · ···· ··· ···········
· ····· ·· ············
····· ········ · ·· ···EE ·
· ······ ······ ·· · LV····· ·····
·· ··· ·DK ·· ··· ···· ··········· ···
···· ··· ··· · · ··· ···· ··· LT ···· ···
· ····· ···· ··· ############ ··················· ···
IE ·· ····GB · ·····##### ## ## ···· BY
· ·· ·· ··· ·NL # ## ··
··· ····· ·· ····· ## # PL ·············
····················BE ··# DE #######····· ····
· ······· ···L## ### CZ ····· ······
······· #### ###·········SK ······· ··
····· ·###### ###### AT ········ ···············
·· FR ··· CH ··#·············· HU ·· MD··
· · ····· · ····SI··· ········· RO ····
· · · ···HR········ RS···· ···
········ ·· · ········ IT ··· ·····BA ·· ··············
· ·· ········· ·········· ·· · · ···ME··XK···· ··
····· ··· · ·· · ··· ···· ·· ··MK·· ·BG ·····
· ·· ····· ··· ····· ···· ·AL············ ······
· ·· ES · · · ·· ····· ···· ···· ······
Coordinates: 51.15°N, 10.55°E
```
## Usage
```
# Auto-detect from IP
which-country-rs
# Specify a country code
which-country-rs -c JP
# Specify coordinates (supports negative values for south/west)
which-country-rs --lat 40 --lon -74
# Custom map size
which-country-rs -W 120 -H 40
```
### Options
| Flag | Description | Default |
|------|-------------|---------|
| `-W, --width` | Map width in characters | 80 |
| `-H, --height` | Map height in characters | 24 |
| `-c, --country` | ISO 3166-1 alpha-2 country code (skips IP lookup) | |
| `--lat` | Latitude (requires `--lon`) | |
| `--lon` | Longitude (requires `--lat`) | |
| `-V, --version` | Print version | |
## Building
```
cargo build --release
```
To build without IP geolocation support (drops the `reqwest` dependency):
```
cargo build --release --no-default-features
```
## Map data
Country borders from [Natural Earth](https://www.naturalearthdata.com/) 110m admin-0 countries (public domain).
File diff suppressed because one or more lines are too long
+25
View File
@@ -0,0 +1,25 @@
{ lib, pkgs, ... }: let
src = ./.;
in
pkgs.rustPlatform.buildRustPackage {
pname = "which-country-rs";
version = "1.0.0";
inherit src;
cargoLock.lockFile = ./Cargo.lock;
# NOTE(yukkop): upstream uses edition 2024, so do not pass the repo's
# Rust 1.81 commonArgs here.
# NOTE(yukkop): keep the Nix package offline-by-default; upstream's
# default geoip feature calls a plaintext HTTP endpoint.
cargoBuildFlags = ["--no-default-features"];
cargoTestFlags = ["--no-default-features"];
meta = {
description = "CLI tool that tells you which country contains coordinates";
homepage = "https://github.com/krisfur/which-country-rs";
license = lib.licenses.mit;
mainProgram = "which-country-rs";
};
}
+275
View File
@@ -0,0 +1,275 @@
use serde::Deserialize;
#[derive(Debug, Deserialize)]
struct FeatureCollection {
features: Vec<Feature>,
}
#[derive(Debug, Deserialize)]
struct Feature {
properties: Properties,
geometry: Geometry,
}
#[derive(Debug, Deserialize)]
struct Properties {
#[serde(rename = "ISO_A2_EH")]
iso_a2: String,
#[serde(rename = "NAME")]
name: String,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
enum Geometry {
Polygon {
coordinates: Vec<Vec<[f64; 2]>>,
},
MultiPolygon {
coordinates: Vec<Vec<Vec<[f64; 2]>>>,
},
}
#[allow(dead_code)]
pub struct Country {
pub iso_a2: String,
pub name: String,
/// Each polygon is a list of rings; ring 0 = outer, rest = holes
pub polygons: Vec<Vec<Vec<[f64; 2]>>>,
pub bbox: (f64, f64, f64, f64), // (min_lon, min_lat, max_lon, max_lat)
pub label_pos: (f64, f64), // (lon, lat) — centroid of largest polygon
}
/// Signed area of a ring (positive = CCW).
fn ring_signed_area(ring: &[[f64; 2]]) -> f64 {
let n = ring.len();
if n < 3 {
return 0.0;
}
let mut area = 0.0;
let mut j = n - 1;
for i in 0..n {
area += (ring[j][0] - ring[i][0]) * (ring[j][1] + ring[i][1]);
j = i;
}
area / 2.0
}
/// Ray-casting point-in-ring test.
fn point_in_ring(lon: f64, lat: f64, ring: &[[f64; 2]]) -> bool {
let mut inside = false;
let n = ring.len();
if n < 3 {
return false;
}
let mut j = n - 1;
for i in 0..n {
let xi = ring[i][0];
let yi = ring[i][1];
let xj = ring[j][0];
let yj = ring[j][1];
if ((yi > lat) != (yj > lat)) && (lon < (xj - xi) * (lat - yi) / (yj - yi) + xi) {
inside = !inside;
}
j = i;
}
inside
}
/// Find horizontal interior spans at a given latitude.
/// Returns sorted pairs of (enter_lon, exit_lon).
fn horizontal_spans(lat: f64, ring: &[[f64; 2]]) -> Vec<(f64, f64)> {
let n = ring.len();
if n < 3 {
return Vec::new();
}
let mut crossings = Vec::new();
let mut j = n - 1;
for i in 0..n {
let yi = ring[i][1];
let yj = ring[j][1];
if (yi > lat) != (yj > lat) {
let xi = ring[i][0];
let xj = ring[j][0];
crossings.push((xj - xi) * (lat - yi) / (yj - yi) + xi);
}
j = i;
}
crossings.sort_by(|a, b| a.partial_cmp(b).unwrap());
crossings.chunks_exact(2).map(|p| (p[0], p[1])).collect()
}
/// Find vertical interior spans at a given longitude.
/// Returns sorted pairs of (enter_lat, exit_lat).
fn vertical_spans(lon: f64, ring: &[[f64; 2]]) -> Vec<(f64, f64)> {
let n = ring.len();
if n < 3 {
return Vec::new();
}
let mut crossings = Vec::new();
let mut j = n - 1;
for i in 0..n {
let xi = ring[i][0];
let xj = ring[j][0];
if (xi > lon) != (xj > lon) {
let yi = ring[i][1];
let yj = ring[j][1];
crossings.push((yj - yi) * (lon - xi) / (xj - xi) + yi);
}
j = i;
}
crossings.sort_by(|a, b| a.partial_cmp(b).unwrap());
crossings.chunks_exact(2).map(|p| (p[0], p[1])).collect()
}
/// Find a good interior label point for a ring.
/// Scans a grid of candidate points and picks the one that maximizes
/// min(half_width, half_height) — the "most interior" point.
fn ring_label_point(ring: &[[f64; 2]]) -> (f64, f64) {
let min_lon = ring.iter().map(|c| c[0]).fold(f64::MAX, f64::min);
let max_lon = ring.iter().map(|c| c[0]).fold(f64::MIN, f64::max);
let min_lat = ring.iter().map(|c| c[1]).fold(f64::MAX, f64::min);
let max_lat = ring.iter().map(|c| c[1]).fold(f64::MIN, f64::max);
let steps = 24;
let mut best = ((min_lon + max_lon) / 2.0, (min_lat + max_lat) / 2.0);
let mut best_score = 0.0f64;
for row in 1..steps {
let lat = min_lat + (max_lat - min_lat) * row as f64 / steps as f64;
let h_spans = horizontal_spans(lat, ring);
for &(span_left, span_right) in &h_spans {
let mid_lon = (span_left + span_right) / 2.0;
let half_w = (span_right - span_left) / 2.0;
// Measure vertical extent at this longitude
let v_spans = vertical_spans(mid_lon, ring);
for &(span_bot, span_top) in &v_spans {
if lat >= span_bot && lat <= span_top {
let half_h = ((lat - span_bot).min(span_top - lat)).min(half_w);
let score = half_w.min(half_h);
if score > best_score {
best_score = score;
best = (mid_lon, lat);
}
break;
}
}
}
}
best
}
pub fn load_countries(geojson: &str) -> Vec<Country> {
let fc: FeatureCollection = serde_json::from_str(geojson).expect("Failed to parse GeoJSON");
fc.features
.into_iter()
.map(|f| {
let polygons = match f.geometry {
Geometry::Polygon { coordinates } => vec![coordinates],
Geometry::MultiPolygon { coordinates } => coordinates,
};
let mut min_lon = f64::MAX;
let mut min_lat = f64::MAX;
let mut max_lon = f64::MIN;
let mut max_lat = f64::MIN;
for poly in &polygons {
for ring in poly {
for coord in ring {
let lon = coord[0];
let lat = coord[1];
if lon < min_lon {
min_lon = lon;
}
if lon > max_lon {
max_lon = lon;
}
if lat < min_lat {
min_lat = lat;
}
if lat > max_lat {
max_lat = lat;
}
}
}
}
// Label inside the largest polygon
let label_pos = polygons
.iter()
.filter(|p| !p.is_empty() && p[0].len() >= 3)
.max_by(|a, b| {
ring_signed_area(&a[0])
.abs()
.partial_cmp(&ring_signed_area(&b[0]).abs())
.unwrap()
})
.map(|p| ring_label_point(&p[0]))
.unwrap_or(((min_lon + max_lon) / 2.0, (min_lat + max_lat) / 2.0));
Country {
iso_a2: f.properties.iso_a2,
name: f.properties.name,
polygons,
bbox: (min_lon, min_lat, max_lon, max_lat),
label_pos,
}
})
.collect()
}
/// Check if a point is inside a polygon (outer ring minus holes).
fn point_in_polygon(lon: f64, lat: f64, rings: &[Vec<[f64; 2]>]) -> bool {
if rings.is_empty() || !point_in_ring(lon, lat, &rings[0]) {
return false;
}
// Must be outside all holes
!rings[1..].iter().any(|hole| point_in_ring(lon, lat, hole))
}
/// Check if a point falls inside a country.
pub fn point_in_country(lon: f64, lat: f64, country: &Country) -> bool {
let (min_lon, min_lat, max_lon, max_lat) = country.bbox;
if lon < min_lon || lon > max_lon || lat < min_lat || lat > max_lat {
return false;
}
country
.polygons
.iter()
.any(|poly| point_in_polygon(lon, lat, poly))
}
/// Find which country contains the given point, with a nearest-country
/// fallback for when low-res coastlines cause a near miss.
pub fn find_country(lon: f64, lat: f64, countries: &[Country]) -> Option<usize> {
// Exact hit
if let Some(idx) = countries.iter().position(|c| point_in_country(lon, lat, c)) {
return Some(idx);
}
// Search in expanding rings up to ~1 degree
for &offset in &[0.25, 0.5, 1.0] {
for &(dlon, dlat) in &[
(offset, 0.0),
(-offset, 0.0),
(0.0, offset),
(0.0, -offset),
(offset, offset),
(offset, -offset),
(-offset, offset),
(-offset, -offset),
] {
if let Some(idx) = countries
.iter()
.position(|c| point_in_country(lon + dlon, lat + dlat, c))
{
return Some(idx);
}
}
}
None
}
+44
View File
@@ -0,0 +1,44 @@
use serde::Deserialize;
#[derive(Debug, Deserialize)]
#[serde(tag = "status")]
pub enum GeoIpResponse {
#[serde(rename = "success")]
Success {
country: String,
#[serde(rename = "countryCode")]
country_code: String,
lat: f64,
lon: f64,
},
#[serde(rename = "fail")]
Fail { message: String },
}
pub struct GeoIpResult {
pub country: String,
pub country_code: String,
pub lat: f64,
pub lon: f64,
}
pub fn lookup() -> Result<GeoIpResult, String> {
let url = "http://ip-api.com/json/?fields=status,message,countryCode,country,lat,lon";
let resp = reqwest::blocking::get(url).map_err(|e| format!("HTTP request failed: {e}"))?;
let data: GeoIpResponse = resp.json().map_err(|e| format!("Failed to parse response: {e}"))?;
match data {
GeoIpResponse::Success {
country,
country_code,
lat,
lon,
} => Ok(GeoIpResult {
country,
country_code,
lat,
lon,
}),
GeoIpResponse::Fail { message } => Err(format!("GeoIP lookup failed: {message}")),
}
}

Some files were not shown because too many files have changed in this diff Show More