Compare commits

67 Commits

Author SHA1 Message Date
yukkop 557b6e9ef0 fix: migrator 2026-07-24 15:21:19 +04:00
yukkop e49f497045 fix: aga 2026-07-24 15:21:11 +04:00
yukkop feb1a48db1 feat: some 2026-07-23 14:19:58 +04:00
yukkop 30732080b7 feat: some 2026-07-21 16:57:10 +04:00
yukkop 80cf1588bb feat: +darwin config for yukkop 2026-07-19 20:03:18 +04:00
yukkop ef7d1b29f4 feat: +njalla module 2026-07-13 23:37:14 +00:00
yukkop 7e8c6884db feat: +plgo 2026-07-13 13:27:04 +00:00
yukkop 1dd41e608b feat: neuro: stable video diffusion 2026-07-05 16:58:47 +00:00
yukkop e41c3e5a05 ssh fixes 2026-07-04 19:32:08 +00:00
yukkop f473280bf5 fix: matrix hardcode 2026-07-01 09:44:43 +00:00
yukkop b08fdd6e6b fix: matrix media 2026-07-01 09:23:26 +00:00
yukkop f73bfc63be fix: zomro: reboot 2026-06-20 22:00:52 +00:00
yukkop f6e7c1eca9 chore 2026-06-11 14:39:57 +00:00
yukkop 6996d178ef fix: hetzner: newer servers generation 2026-06-10 15:39:05 +00:00
yukkop 7f7229b199 feat: tenix host 2026-06-10 13:15:09 +00:00
yukkop 2d5bd26c36 docs: migrator: ~ logs & comments 2026-06-10 12:26:16 +00:00
yukkop fba150b55b docs: db-tool: ~ postgres-init and postgres-cleanup 2026-06-10 12:14:23 +00:00
yukkop 2e7bf58acf refactor: db-tool: rename file 2026-06-10 11:59:21 +00:00
yukkop b12c35f957 fix: db-tool 2026-06-10 11:40:35 +00:00
yukkop bcf1b84dc4 fix: db-tool 2026-06-10 11:32:37 +00:00
yukkop 7c25e3b46d feat: db-tool: +secrets load 2026-06-09 23:57:51 +00:00
yukkop a20381e343 chore: ssh key 2026-06-09 22:35:06 +00:00
yukkop bd92610a98 feat: floating ip 2026-06-09 15:43:23 +00:00
yukkop e3ee881db6 chore: lab: neuro's ssh key 2026-06-09 15:11:41 +00:00
yukkop b9eabca464 feat: load-sops: +generic 2026-06-09 14:51:11 +00:00
yukkop fcc72192f5 dev: gitea: runners devshell 2026-06-08 10:22:34 +00:00
yukkop 5a0696ce64 ci: gitea: runners infra 2026-06-08 08:18:08 +00:00
yukkop f4a59ff117 fix: mechabellum: 413 2026-06-07 23:14:14 +00:00
yukkop 129c82c863 chore: update mechabellum 2026-06-07 22:38:54 +00:00
yukkop 968c654320 fix: gitea: timeouts per iphone 429 errors 2026-06-07 22:24:11 +00:00
yukkop 98bb6c568f chore(package): gitea: verify heatmap package build 2026-06-06 23:01:57 +00:00
yukkop 72168aa8fa test(package): gitea: verify heatmap privacy endpoints
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-06 22:26:43 +00:00
yukkop 28dde5b9b1 test(package): gitea: cover heatmap author-date semantics
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus-Junior (openai/gpt-5.5) <clio-agent@sisyphuslabs.ai>
2026-06-06 22:10:18 +00:00
yukkop c2e0ba200c feat(package): gitea: wire heatmap reindexing
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-06 22:01:59 +00:00
yukkop 2eb23ea7ea feat(package): gitea: query heatmap by commit author date 2026-06-06 20:22:12 +00:00
yukkop 9906f71c5d feat(package): gitea: add private heatmap opt-in 2026-06-06 20:06:21 +00:00
yukkop 593c0d9abc feat(package): gitea: index heatmap commits by author date 2026-06-06 19:44:53 +00:00
yukkop 53dfd60b4c feat(package): gitea: add heatmap contribution model 2026-06-06 19:20:37 +00:00
yukkop 3299daf061 feat: db-tool: update hook realization 2026-06-06 18:17:16 +00:00
yukkop 2856ca1d98 feat: ente: smpt increase security level 2026-06-06 15:34:47 +00:00
yukkop e04b7e11da feat: ente: added 2026-06-06 13:26:36 +00:00
yukkop 59dc5ecd1e feat: elment: enable video messages 2026-06-06 11:09:49 +00:00
yukkop ad6c5ab803 feat: base: +cache 2026-06-06 11:04:13 +00:00
yukkop 592d1f04c5 feat: element-web: video messages 2026-06-06 05:25:02 +00:00
yukkop d76c0b0273 feat: vendor element-web 2026-06-05 18:55:22 +00:00
yukkop 0914391a2b docs: atticd cache 2026-06-05 13:42:17 +00:00
yukkop d88c1cbb4f chore: update inputs 2026-06-05 13:12:22 +00:00
yukkop 35a5d59cbe feat: caching 2026-06-05 13:09:18 +00:00
yukkop 341e3a0e1c feat: atticd to S3 2026-06-05 12:42:55 +00:00
yukkop a30d1a93dd fix: element things 2026-06-05 10:55:51 +00:00
yukkop c50d274ae1 feat: lab: +attic cache 2026-06-04 18:58:03 +00:00
yukkop df19d16269 chore: lab: evgenii-kazakov email 2026-06-04 18:10:09 +00:00
yukkop 35af6720ef fix 2026-06-03 15:03:05 +00:00
yukkop a33432d5de fix 2026-06-03 14:55:20 +00:00
yukkop 7152eb03de fix 2026-06-03 13:03:29 +00:00
yukkop 8e5ba8de7f fix 2026-06-03 12:37:44 +00:00
yukkop 7c10fda451 fix 2026-06-03 12:22:42 +00:00
yukkop ca88f92b1a fix 2026-06-03 09:19:12 +00:00
yukkop 5b5f119a65 fix 2026-06-03 09:08:05 +00:00
yukkop 8caf575946 fix 2026-06-03 08:54:08 +00:00
yukkop d644d390a7 fix 2026-06-03 08:36:42 +00:00
yukkop b56dc50e50 feat: db-tool: normalize-backup 2026-06-03 08:14:22 +00:00
yukkop 63223329dd chore 2026-06-03 04:50:53 +00:00
yukkop c74992ea85 fix: db-tool: generic for dbname in diff 2026-06-02 20:33:52 +00:00
yukkop c0c024dcfd fix: lab: mechabellum ssl 2026-06-02 20:33:19 +00:00
yukkop 0023e27110 feat: lab: disable gitea auth 2026-06-02 20:32:48 +00:00
yukkop 882b4ec871 feat: db-tool: fail on migration error in diff 2026-06-02 19:38:59 +00:00
133 changed files with 10856 additions and 686 deletions
+29
View File
@@ -0,0 +1,29 @@
name: runner nix smoke
on:
workflow_dispatch:
push:
branches:
- master
paths:
- .gitea/workflows/runner-nix-smoke.yaml
- flake.lock
- flake.nix
- infra/gitea-runners/**
jobs:
smoke:
name: nix label smoke
runs-on: nix
steps:
- name: Nix version and cache configuration
run: |
set -eu
nix --version
nix config show substituters
nix config show trusted-public-keys
- name: Repository flake evaluation
run: |
set -eu
nix flake check --no-build
+31
View File
@@ -0,0 +1,31 @@
name: runner ubuntu smoke
on:
workflow_dispatch:
push:
branches:
- master
paths:
- .gitea/workflows/runner-ubuntu-smoke.yaml
- infra/gitea-runners/**
jobs:
smoke:
name: ubuntu-latest label smoke
runs-on: ubuntu-latest
steps:
- name: Basic runner information
run: |
set -eu
echo "hello from gitea runner"
uname -a
- name: Docker smoke when available
run: |
set -eu
if command -v docker >/dev/null 2>&1; then
docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
docker run --rm hello-world
else
echo "docker command not available; skipping Docker smoke"
fi
+9
View File
@@ -47,6 +47,15 @@ creation_rules:
- *hectic-lab-server - *hectic-lab-server
- *umbriel-bfs - *umbriel-bfs
- path_regex: sus/gitea-runners.yaml$
key_groups:
- age:
- *nrv
- *yukkop
- *yukkop-alt
- *hectic-lab-server
- *umbriel-bfs
- path_regex: sus/matrix-cluster.yaml$ - path_regex: sus/matrix-cluster.yaml$
key_groups: key_groups:
- age: - age:
+21
View File
@@ -0,0 +1,21 @@
{
flake,
self,
inputs,
system ? "aarch64-darwin",
...
}: let
name = builtins.baseNameOf ./.;
in inputs.nix-darwin.lib.darwinSystem {
inherit system;
specialArgs = { inherit flake self inputs; };
modules = [
inputs.home-manager.darwinModules.home-manager
{
networking.hostName = name;
nixpkgs.hostPlatform = system;
nixpkgs.overlays = [ self.overlays.default ];
}
./${name}.nix
];
}
+177
View File
@@ -0,0 +1,177 @@
{
flake,
pkgs,
lib,
...
}: let
name = "yukkop";
in {
system.primaryUser = name;
nix.settings.experimental-features = "nix-command flakes";
programs.zsh.enable = true;
services.openssh.enable = true;
users.users.${name} = {
home = "/Users/${name}";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
];
};
environment.systemPackages = with pkgs; [
aerospace
git
moreutils
neovim
tmux
];
launchd.user.agents.aerospace = {
serviceConfig = {
ProgramArguments = [
"${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
];
RunAtLoad = true;
KeepAlive = true;
StandardOutPath = "/tmp/aerospace.out.log";
StandardErrorPath = "/tmp/aerospace.err.log";
};
};
home-manager.useGlobalPkgs = true;
home-manager.useUserPackages = true;
home-manager.backupFileExtension = "backup";
home-manager.sharedModules = [
(flake + "/home/module/program/tmux.nix")
];
home-manager.users.${name} = {
home.stateVersion = "25.11";
home.packages = with pkgs; [
iproute2mac
jujutsu
ripgrep
];
programs.git = {
enable = true;
lfs.enable = true;
settings = {
user.name = name;
user.email = "hectic.yukkop@gmail.com";
push.autoSetupRemote = true;
init.defaultBranch = "master";
};
};
programs.zsh = {
enable = true;
enableCompletion = true;
autosuggestion.enable = true;
syntaxHighlighting.enable = true;
history = {
size = 10000;
path = "$HOME/.zsh/.zsh_history";
};
shellAliases = {
drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
nv = "nvim";
tmux = "tmux a";
};
initContent = ''
export PATH=/Users/yukkop/.opencode/bin:$PATH
'';
};
xdg.configFile."aerospace/aerospace.toml".text = ''
start-at-login = false
enable-normalization-flatten-containers = true
enable-normalization-opposite-orientation-for-nested-containers = true
default-root-container-layout = 'tiles'
default-root-container-orientation = 'auto'
accordion-padding = 30
on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
automatically-unhide-macos-hidden-apps = false
[exec]
inherit-env-vars = true
[exec.env-vars]
PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
[gaps]
inner.horizontal = 8
inner.vertical = 8
outer.left = 8
outer.bottom = 8
outer.top = 8
outer.right = 8
[mode.main.binding]
alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
alt-slash = 'layout tiles horizontal vertical'
alt-comma = 'layout accordion horizontal vertical'
alt-f = 'fullscreen'
alt-h = 'focus left'
alt-j = 'focus down'
alt-k = 'focus up'
alt-l = 'focus right'
alt-shift-h = 'move left'
alt-shift-j = 'move down'
alt-shift-k = 'move up'
alt-shift-l = 'move right'
alt-minus = 'resize smart -50'
alt-equal = 'resize smart +50'
alt-1 = 'workspace 1'
alt-2 = 'workspace 2'
alt-3 = 'workspace 3'
alt-4 = 'workspace 4'
alt-5 = 'workspace 5'
alt-6 = 'workspace 6'
alt-7 = 'workspace 7'
alt-8 = 'workspace 8'
alt-9 = 'workspace 9'
alt-shift-1 = 'move-node-to-workspace 1'
alt-shift-2 = 'move-node-to-workspace 2'
alt-shift-3 = 'move-node-to-workspace 3'
alt-shift-4 = 'move-node-to-workspace 4'
alt-shift-5 = 'move-node-to-workspace 5'
alt-shift-6 = 'move-node-to-workspace 6'
alt-shift-7 = 'move-node-to-workspace 7'
alt-shift-8 = 'move-node-to-workspace 8'
alt-shift-9 = 'move-node-to-workspace 9'
alt-tab = 'workspace-back-and-forth'
alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
alt-shift-semicolon = 'mode service'
[mode.service.binding]
esc = ['reload-config', 'mode main']
r = ['flatten-workspace-tree', 'mode main']
f = ['layout floating tiling', 'mode main']
b = ['balance-sizes', 'mode main']
backspace = ['close-all-windows-but-current', 'mode main']
alt-shift-h = ['join-with left', 'mode main']
alt-shift-j = ['join-with down', 'mode main']
alt-shift-k = ['join-with up', 'mode main']
alt-shift-l = ['join-with right', 'mode main']
'';
};
system.stateVersion = 6;
}
+1
View File
@@ -8,6 +8,7 @@
haskell = import ./haskell.nix { inherit self system pkgs; }; haskell = import ./haskell.nix { inherit self system pkgs; };
neuro = import ./neuro.nix { inherit self system pkgs; }; neuro = import ./neuro.nix { inherit self system pkgs; };
xmpp = import ./xmpp.nix { inherit self system pkgs; }; xmpp = import ./xmpp.nix { inherit self system pkgs; };
gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
default = pkgs.mkShell { default = pkgs.mkShell {
buildInputs = buildInputs =
(with self.packages.${system}; [ (with self.packages.${system}; [
+122
View File
@@ -0,0 +1,122 @@
{ pkgs, ... }: let
opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
tofu = pkgs.writeShellScriptBin "tofu" ''
exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
'';
giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
cat <<'EOF'
Gitea runners setup checklist
Tools available in this shell:
tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
Environment expected before real deploy/apply:
TF_VAR_hcloud_token
TF_VAR_ssh_public_key
TF_VAR_ssh_private_key
S3 backend credentials and endpoint access
a matching SOPS age identity for sus/gitea-runners.yaml
kubectl access to the target cluster
a concrete registry digest for the pushed Nix-capable runner image if enabling
the nix label
OpenTofu validation gate:
tofu version
tofu -chdir=infra/gitea-runners/opentofu validate
Nix image build/publish/digest gate:
nix build .#gitea-runner-nix-image
publish the archive, then pin the registry-reported digest in the runner label
mapping
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
SOPS token Secret creation gate:
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
umask 077
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
trap 'rm -f "$token_file"' EXIT
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
kubectl -n gitea-runners create secret generic gitea-runner-token \
--from-file=token="$token_file" \
--dry-run=client \
-o yaml | kubectl -n gitea-runners apply -f -
Cluster provision gate:
tofu -chdir=infra/gitea-runners/opentofu init
tofu -chdir=infra/gitea-runners/opentofu validate
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
Kubernetes apply gate:
kubectl config current-context
kubectl get nodes -o wide
kubectl get sc
kubectl apply -k infra/gitea-runners/k8s
Verification commands:
kubectl -n gitea-runners get statefulset gitea-runner
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
Main blockers and gates:
do not run tofu apply without all external inputs
do not apply the k8s overlay until the gitea-runner-token Secret exists
do not enable the nix label until the image has been published with a concrete digest
do not print, load, or require secrets on shell entry
EOF
'';
in pkgs.mkShell {
name = "gitea-runners";
buildInputs = [
tofu
giteaRunnersSetup
pkgs.nix
pkgs.kubectl
pkgs.kustomize
pkgs.kubeconform
pkgs.sops
pkgs.age
pkgs.awscli2
pkgs.hcloud
pkgs.tea
pkgs.docker
pkgs.skopeo
pkgs.go-containerregistry
pkgs.jq
pkgs.yq-go
pkgs.curl
pkgs.git
pkgs.openssh
];
shellHook = ''
export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
export GITEA_RUNNERS_NAMESPACE="gitea-runners"
alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
echo ""
echo "=== Gitea runner setup DevShell ==="
echo ""
echo "Run gitea-runners-setup for the full setup checklist."
echo "Paths: "
echo " root=$GITEA_RUNNERS_ROOT"
echo " tofu=$GITEA_RUNNERS_TOFU_DIR"
echo " k8s=$GITEA_RUNNERS_K8S_DIR"
echo " image=$GITEA_RUNNERS_IMAGE_DIR"
echo ""
'';
}
+3
View File
@@ -0,0 +1,3 @@
# Documentation
- [Using the `hectic` Attic Cache](./attic-cache.md)
+237
View File
@@ -0,0 +1,237 @@
# Using the `hectic` Attic Cache
This document explains how to:
1. pull build artifacts from the cache
2. push new artifacts to the cache
3. configure this flake to use the cache
## Cache endpoints
- API endpoint: `https://cache.hectic-lab.com`
- Binary cache endpoint: `https://cache.hectic-lab.com/hectic`
The `hectic` cache is:
- public for reads
- private for pushes
## Requirements
Use the Attic client package:
```sh
nix shell nixpkgs#attic-client
```
Or run commands directly with:
```sh
nix shell nixpkgs#attic-client -c <command>
```
## Read from the cache
### Get the cache public key
```sh
nix shell nixpkgs#attic-client -c attic cache info hectic
```
Copy the `Public Key` value, which looks like:
```text
hectic:...
```
### Configure Nix to trust the cache
Per-user: `~/.config/nix/nix.conf`
```ini
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
```
System-wide: `/etc/nix/nix.conf`
```ini
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
```
After that, normal Nix commands can download from the cache automatically:
```sh
nix build .#migrator
nix develop
nix flake check
```
## Use the cache from this flake
You can also advertise the cache from `flake.nix`:
```nix
nixConfig = {
extra-substituters = [
"https://cache.nixos.org"
"https://cache.hectic-lab.com/hectic"
];
extra-trusted-public-keys = [
"hectic:PASTE_PUBLIC_KEY_HERE"
];
};
```
Then users can run:
```sh
nix build --accept-flake-config .#migrator
```
## Log in for pushing
Pushing requires an Attic token.
```sh
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
```
Example with `pass`:
```sh
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "$(pass show atticd/hectic-lab/token)"
```
## Push build results
### Push a package
```sh
nix build .#migrator
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
```
### Push a check
```sh
nix build .#checks.x86_64-linux.arguments
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
```
### Push a NixOS system build
```sh
nix build '.#nixosConfigurations."hectic-lab|x86_64-linux".config.system.build.toplevel'
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
```
## Recommended workflow
### Local development
Use the cache for reads only:
```sh
nix build .#migrator
nix develop
nix flake check
```
### CI / builder
1. Build
2. Push to Attic
Example:
```sh
nix build .#migrator
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
```
## Useful commands
### Show cache info
```sh
nix shell nixpkgs#attic-client -c attic cache info hectic
```
### Check login config
```sh
nix shell nixpkgs#attic-client -c attic cache info local:hectic
```
### Re-login with a new token
```sh
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<NEW_TOKEN>"
```
## Common issues
### `flake 'nixpkgs' does not provide attribute 'attic'`
Use:
```sh
nix shell nixpkgs#attic-client
```
Not:
```sh
nix shell nixpkgs#attic
```
### `HTTP 413 Payload Too Large`
This means nginx rejected the upload body size. The server must allow large uploads on the Attic vhost.
### Push succeeds for some paths but fails for others
Usually means:
- nginx body size limit
- timeout/reverse proxy issue
- bad token permissions
### Cache pulls do not work
Check:
- `substituters`
- `trusted-public-keys`
- the exact public key from `attic cache info hectic`
## Notes about retention and storage
- The cache currently uses Hetzner Object Storage
- If no `retention-period` is configured, cached objects do not expire automatically
- This is good for long-lived reuse, but storage usage can grow over time
## Summary
### Read access
```sh
nix build .#migrator
```
after configuring:
```ini
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
```
### Push access
```sh
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
nix build .#migrator
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
```
Generated
+36 -31
View File
@@ -326,7 +326,7 @@
"hectic-landing": { "hectic-landing": {
"inputs": { "inputs": {
"nixpkgs": [ "nixpkgs": [
"nixpkgs-fixed" "nixpkgs"
] ]
}, },
"locked": { "locked": {
@@ -346,7 +346,7 @@
"home-manager": { "home-manager": {
"inputs": { "inputs": {
"nixpkgs": [ "nixpkgs": [
"nixpkgs-fixed" "nixpkgs"
] ]
}, },
"locked": { "locked": {
@@ -671,15 +671,15 @@
"mechabellum-replay-analysis": { "mechabellum-replay-analysis": {
"inputs": { "inputs": {
"nixpkgs": [ "nixpkgs": [
"nixpkgs-fixed" "nixpkgs"
] ]
}, },
"locked": { "locked": {
"lastModified": 1779576166, "lastModified": 1780905541,
"narHash": "sha256-5bSuXkQs7KdbaYwDTdwUFlqOccVjPI2y42TZVq8lsNg=", "narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
"ref": "refs/heads/master", "ref": "refs/heads/master",
"rev": "f00295225c0dade61fe18b32262970c2665fb5fe", "rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
"revCount": 110, "revCount": 123,
"type": "git", "type": "git",
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git" "url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
}, },
@@ -688,6 +688,27 @@
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git" "url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
} }
}, },
"nix-darwin": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1772129556,
"narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
"owner": "nix-darwin",
"repo": "nix-darwin",
"rev": "ebec37af18215214173c98cf6356d0aca24a2585",
"type": "github"
},
"original": {
"owner": "nix-darwin",
"ref": "nix-darwin-25.11",
"repo": "nix-darwin",
"type": "github"
}
},
"nix-minecraft": { "nix-minecraft": {
"inputs": { "inputs": {
"flake-compat": "flake-compat_3", "flake-compat": "flake-compat_3",
@@ -872,29 +893,13 @@
"type": "github" "type": "github"
} }
}, },
"nixpkgs-fixed": {
"locked": {
"lastModified": 1771419570,
"narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-25.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": { "nixpkgs_2": {
"locked": { "locked": {
"lastModified": 1771419570, "lastModified": 1779796641,
"narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=", "narHash": "sha256-ZsIrKmhp4vbBXoXXmR/tBXA/UCsAQiJL9vsgZEduhVY=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47", "rev": "25f538306313eae3927264466c70d7001dcea1df",
"type": "github" "type": "github"
}, },
"original": { "original": {
@@ -908,7 +913,7 @@
"inputs": { "inputs": {
"flake-parts": "flake-parts_2", "flake-parts": "flake-parts_2",
"nixpkgs": [ "nixpkgs": [
"nixpkgs-fixed" "nixpkgs"
], ],
"nuschtosSearch": "nuschtosSearch", "nuschtosSearch": "nuschtosSearch",
"systems": "systems_5" "systems": "systems_5"
@@ -983,13 +988,13 @@
"hyprland": "hyprland", "hyprland": "hyprland",
"impermanence": "impermanence", "impermanence": "impermanence",
"mechabellum-replay-analysis": "mechabellum-replay-analysis", "mechabellum-replay-analysis": "mechabellum-replay-analysis",
"nix-darwin": "nix-darwin",
"nix-minecraft": "nix-minecraft", "nix-minecraft": "nix-minecraft",
"nixos-anywhere": "nixos-anywhere", "nixos-anywhere": "nixos-anywhere",
"nixos-hardware": "nixos-hardware", "nixos-hardware": "nixos-hardware",
"nixos-mailserver": "nixos-mailserver", "nixos-mailserver": "nixos-mailserver",
"nixos-wsl": "nixos-wsl", "nixos-wsl": "nixos-wsl",
"nixpkgs": "nixpkgs_2", "nixpkgs": "nixpkgs_2",
"nixpkgs-fixed": "nixpkgs-fixed",
"nixvim": "nixvim", "nixvim": "nixvim",
"rust-overlay": "rust-overlay", "rust-overlay": "rust-overlay",
"sops-nix": "sops-nix" "sops-nix": "sops-nix"
@@ -1002,11 +1007,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1738290352, "lastModified": 1780629589,
"narHash": "sha256-YKOHUmc0Clm4tMV8grnxYL4IIwtjTayoq/3nqk0QM7k=", "narHash": "sha256-oHysjxZdaEqkmyDpN8G1bl3V+r9uyRD1O66bH0bq0Cs=",
"owner": "oxalica", "owner": "oxalica",
"repo": "rust-overlay", "repo": "rust-overlay",
"rev": "b031b584125d33d23a0182f91ddbaf3ab4880236", "rev": "7a5a1c0a5cb86a28224304309b68f050835fd1f6",
"type": "github" "type": "github"
}, },
"original": { "original": {
+25 -7
View File
@@ -1,8 +1,18 @@
{ {
description = "yukkop's nix utilities"; description = "yukkop's nix utilities";
nixConfig = {
extra-substituters = [
"https://cache.nixos.org"
"https://cache.hectic-lab.com/hectic"
];
extra-trusted-public-keys = [
"hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
];
};
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
nixpkgs-fixed.url = "github:NixOS/nixpkgs/nixos-25.11";
rust-overlay = { rust-overlay = {
url = "github:oxalica/rust-overlay"; url = "github:oxalica/rust-overlay";
inputs = { inputs = {
@@ -19,7 +29,7 @@
}; };
nixvim = { nixvim = {
url = "github:nix-community/nixvim/nixos-25.11"; url = "github:nix-community/nixvim/nixos-25.11";
inputs.nixpkgs.follows = "nixpkgs-fixed"; inputs.nixpkgs.follows = "nixpkgs";
}; };
disko = { disko = {
url = "github:nix-community/disko"; url = "github:nix-community/disko";
@@ -30,7 +40,11 @@
}; };
home-manager = { home-manager = {
url = "github:nix-community/home-manager/release-25.11"; url = "github:nix-community/home-manager/release-25.11";
inputs.nixpkgs.follows = "nixpkgs-fixed"; inputs.nixpkgs.follows = "nixpkgs";
};
nix-darwin = {
url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
inputs.nixpkgs.follows = "nixpkgs";
}; };
nixos-wsl = { nixos-wsl = {
url = "github:nix-community/NixOS-WSL"; url = "github:nix-community/NixOS-WSL";
@@ -56,13 +70,13 @@
# NOTE(yukkop): private repo - SSH access required. # NOTE(yukkop): private repo - SSH access required.
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built. # Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
url = "git+ssh://git@github.com/liquizz/hectic-landing.git"; url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
inputs.nixpkgs.follows = "nixpkgs-fixed"; inputs.nixpkgs.follows = "nixpkgs";
}; };
mechabellum-replay-analysis = { mechabellum-replay-analysis = {
# NOTE(yukkop): private repo - SSH access required. # NOTE(yukkop): private repo - SSH access required.
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built. # Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"; url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git";
inputs.nixpkgs.follows = "nixpkgs-fixed"; inputs.nixpkgs.follows = "nixpkgs";
}; };
}; };
@@ -109,8 +123,12 @@
# FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this # FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; }; "neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; }; "games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; }; "wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; }; "tenix|x86_64-linux" = import ./nixos/system/tenix { inherit flake self inputs; system = "x86_64-linux"; };
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
};
darwinConfigurations = {
"yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
}; };
}; };
} }
+30
View File
@@ -0,0 +1,30 @@
{ pkgs, ... }: {
programs.tmux = {
enable = true;
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
keyMode = "vi";
escapeTime = 500;
historyLimit = 50000;
newSession = true;
extraConfig = ''
# resurrect
set -g @resurrect-strategy-vim 'session'
set -g @resurrect-strategy-nvim 'session'
set -g @resurrect-capture-pane-contents 'on'
resurrect_dir="$HOME/.tmux/resurrect"
set -g @resurrect-dir $resurrect_dir
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
# continuum
set -g @continuum-restore 'on'
set -g @continuum-boot 'on'
set -g @continuum-save-interval '10'
bind-key -T copy-mode-vi v send-keys -X begin-selection
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
bind-key O select-pane -t :.-
'';
};
}
+80
View File
@@ -0,0 +1,80 @@
# Gitea runner Nix image
The repo-owned Nix-capable job image is built by the flake package
`gitea-runner-nix-image`.
```sh
nix build .#gitea-runner-nix-image
```
The package emits a Docker archive with the local build tag:
```text
gitea-runner-nix-image:2026-06-07
```
That tag is build metadata only. Do not use it as the final Gitea runner label
mapping because runner job images must be immutable.
## Publication target
Preferred registry:
```text
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
```
Publish the archive without adding secrets to the image layers, then use the
registry-reported digest as the only final `nix` label image reference:
```text
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
```
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
runner label mapping must use the `@sha256:` reference above. Keep
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
configuration task explicitly changes it. Only the `nix` label should select
this custom image.
If the Gitea container registry is unavailable, select a private registry that
is reachable from the runner Kubernetes cluster and requires authentication that
can be provided through Kubernetes image-pull secrets. Record the selected
registry and replace the host in the same digest-pinned form:
```text
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
```
Do not fall back to `latest` or a tag-only mapping.
## Task 7 publication status
Local build evidence is recorded in
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
load and tag the image, but pushing to the preferred registry failed with
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
concrete final mapping. Kubernetes pull smoke is recorded in
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
is not installed or not on `PATH`.
Once registry credentials are available, rerun the push, capture the
registry-reported digest, and replace `<registry-digest>` in the mapping above
before Task 6/9 consumes the label configuration.
## Image contents
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
top-level `flake.nix`:
```text
experimental-features = nix-command flakes
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
sandbox = false
```
No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
credential belongs in this image. Runtime secrets stay with the Kubernetes
runner configuration and token-file mount contract.
@@ -0,0 +1,154 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
rules:
- apiGroups:
- ""
resources:
- pods
- persistentvolumeclaims
verbs:
- get
- list
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
subjects:
- kind: ServiceAccount
name: gitea-runner-lifecycle
namespace: gitea-runners
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: gitea-runner-lifecycle
---
apiVersion: v1
kind: ConfigMap
metadata:
name: gitea-runner-lifecycle
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
data:
cleanup-dry-run.sh: |
#!/bin/sh
set -eu
namespace="${RUNNER_NAMESPACE:-gitea-runners}"
mode="${CLEANUP_MODE:-dry-run}"
selector="app.kubernetes.io/name=gitea-runner"
if [ "$namespace" != "gitea-runners" ]; then
printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
exit 13
fi
if [ "$mode" != "dry-run" ]; then
printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
exit 13
fi
printf 'gitea runner lifecycle cleanup dry-run\n'
printf 'namespace: %s\n' "$namespace"
printf 'mode: %s\n\n' "$mode"
printf 'StatefulSet:\n'
kubectl -n "$namespace" get statefulset gitea-runner -o wide
printf '\nActive runner pods:\n'
kubectl -n "$namespace" get pods -l "$selector" -o wide
printf '\nRunner /data PVCs:\n'
kubectl -n "$namespace" get pvc -l "$selector" -o wide
printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
found_candidate=0
for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
pod="${pvc#data-}"
if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
found_candidate=1
printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
fi
done
if [ "$found_candidate" -eq 0 ]; then
printf 'none\n'
fi
printf '\nGitea registration reconciliation:\n'
printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: gitea-runner-cleanup-dry-run
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
spec:
schedule: "17 3 * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
ttlSecondsAfterFinished: 3600
template:
metadata:
labels:
app.kubernetes.io/name: gitea-runner-lifecycle
app.kubernetes.io/part-of: gitea-actions
spec:
serviceAccountName: gitea-runner-lifecycle
restartPolicy: Never
containers:
- name: cleanup-dry-run
image: bitnami/kubectl:1.30
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- /scripts/cleanup-dry-run.sh
env:
- name: RUNNER_NAMESPACE
value: gitea-runners
- name: CLEANUP_MODE
value: dry-run
volumeMounts:
- name: lifecycle-scripts
mountPath: /scripts
readOnly: true
volumes:
- name: lifecycle-scripts
configMap:
name: gitea-runner-lifecycle
defaultMode: 0555
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- service.yaml
- service-account.yaml
- runner-config.yaml
- statefulset.yaml
- cleanup-lifecycle.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
@@ -0,0 +1,36 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: gitea-runner-config
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
data:
config.yaml: |
log:
level: info
runner:
file: /data/.runner
capacity: 1
envs: {}
timeout: 3h
insecure: false
fetch_timeout: 5s
fetch_interval: 2s
labels:
- ubuntu-latest
# The nix label is intentionally disabled until the runner image has a
# concrete registry-reported digest; see ../runbook.md before deploy.
cache:
enabled: true
dir: /data/cache
container:
network: bridge
privileged: false
force_pull: true
valid_volumes: []
docker_host: unix:///runner-docker/docker.sock
@@ -0,0 +1,36 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
rules: []
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
subjects:
- kind: ServiceAccount
name: gitea-runner
namespace: gitea-runners
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: gitea-runner
+16
View File
@@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
spec:
clusterIP: None
selector:
app.kubernetes.io/name: gitea-runner
ports:
- name: cache
port: 8088
targetPort: cache
+156
View File
@@ -0,0 +1,156 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: gitea-runner
namespace: gitea-runners
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
spec:
serviceName: gitea-runner
replicas: 5
podManagementPolicy: Parallel
selector:
matchLabels:
app.kubernetes.io/name: gitea-runner
template:
metadata:
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
annotations:
hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
spec:
serviceAccountName: gitea-runner
automountServiceAccountToken: false
terminationGracePeriodSeconds: 60
securityContext:
fsGroup: 1000
containers:
- name: runner
image: gitea/act_runner:0.2.11
imagePullPolicy: IfNotPresent
env:
- name: GITEA_INSTANCE_URL
value: https://gitea.hectic-lab.com
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
value: /runner-secrets/token
- name: CONFIG_FILE
value: /runner-config/config.yaml
- name: DOCKER_HOST
value: unix:///runner-docker/docker.sock
ports:
- name: cache
containerPort: 8088
resources:
requests:
cpu: 250m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
exec:
command:
- /bin/sh
- -ec
- test -s /data/.runner && test -S /runner-docker/docker.sock
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
exec:
command:
- /bin/sh
- -ec
- test -S /runner-docker/docker.sock
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
volumeMounts:
- name: data
mountPath: /data
- name: config
mountPath: /runner-config
readOnly: true
- name: runner-token
mountPath: /runner-secrets
readOnly: true
- name: docker-socket
mountPath: /runner-docker
- name: docker
image: docker:27-dind
imagePullPolicy: IfNotPresent
args:
- --host=unix:///runner-docker/docker.sock
- --storage-driver=overlay2
- --tls=false
env:
- name: DOCKER_TLS_CERTDIR
value: ""
- name: DOCKER_HOST
value: unix:///runner-docker/docker.sock
securityContext:
# Privileged rootful DinD is intentionally scoped to this trusted
# internal runner pool; never expose it to untrusted fork/PR jobs.
privileged: true
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
exec:
command:
- docker
- info
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 6
readinessProbe:
exec:
command:
- docker
- info
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 6
volumeMounts:
- name: docker-socket
mountPath: /runner-docker
- name: docker-graph
mountPath: /var/lib/docker
volumes:
- name: config
configMap:
name: gitea-runner-config
- name: runner-token
secret:
secretName: gitea-runner-token
items:
- key: token
path: token
defaultMode: 0400
- name: docker-socket
emptyDir: {}
- name: docker-graph
emptyDir: {}
volumeClaimTemplates:
- metadata:
name: data
labels:
app.kubernetes.io/name: gitea-runner
app.kubernetes.io/part-of: gitea-actions
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes
resources:
requests:
storage: 20Gi
+29
View File
@@ -0,0 +1,29 @@
# OpenTofu working directory and downloaded modules/providers.
.terraform/
.terraform.lock.hcl
# State must live in the S3 backend for production. Local state is allowed only
# for throwaway syntax checks with `tofu init -backend=false` and must not be
# committed.
terraform.tfstate
terraform.tfstate.*
*.tfstate
*.tfstate.*
crash.log
crash.*.log
# Plans can contain secrets or derived infrastructure data.
*.tfplan
*.plan
kubeconfig
kubeconfig.yaml
*_kubeconfig.yaml
# Variable files commonly carry credentials. Keep production inputs in SOPS or
# external environment/configuration, not in checked-in files.
*.tfvars
*.tfvars.json
override.tf
override.tf.json
*_override.tf
*_override.tf.json
+90
View File
@@ -0,0 +1,90 @@
# Gitea runner OpenTofu backend contract
This directory defines the safe backend, provider contract, and kube-hetzner
cluster stack for the Gitea runner Kubernetes cluster.
## Required backend
Production state must use the OpenTofu S3 backend in `backend.tf`:
- bucket: `gitea-runner-hectic-lab`
- key: `gitea-runners/kube-hetzner/terraform.tfstate`
- region: `fsn1`, aligned with the target Hetzner location
- encryption: `encrypt = true`
- locking: `use_lockfile = true` where the selected S3-compatible endpoint
supports it
Before any production `tofu init`, verify the S3-compatible endpoint, credential
source, bucket versioning, encryption behavior, and lockfile support for the
chosen object-storage provider. Keep backend authentication externalized through
environment variables, AWS-compatible shared config, or the production secret
injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
OpenTofu files.
## Local state safety
Production local state is forbidden. Only syntax-only validation/prototyping may
use local state, and it must use backend-disabled initialization:
```sh
tofu -chdir=infra/gitea-runners/opentofu init -backend=false
tofu -chdir=infra/gitea-runners/opentofu validate
```
Fail the run if production local state appears:
```sh
test ! -e infra/gitea-runners/opentofu/terraform.tfstate
test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
grep -R 'backend "s3"' infra/gitea-runners/opentofu
```
The `.gitignore` in this directory blocks local state, plans, downloaded
providers/modules, and variable files from being committed. Treat any local
state file as disposable validation residue, never as production state.
## Provider and module pins
`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
as locals so Task 5 can wire the module without re-opening the version contract.
`providers.tf` leaves the `hcloud` provider empty so authentication comes from
the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
Do not set token values in `.tf` or `.tfvars` files.
## Cluster shape
The default cluster is deliberately fixed-size:
- cluster name: `gitea-runners`
- Hetzner location: `fsn1`
- private network region: `eu-central`
- control plane: one `cpx21` node in pool `control-plane`
- workers: three `cpx31` nodes in pool `runner-workers`
- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
- Longhorn: disabled
- autoscaling/KEDA: not enabled in this stack
The three default workers are sized for the initial five trusted privileged DinD
jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
`worker_count` to `5` or increase `worker_server_type`, then run a fresh
`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
Required inputs must come from environment or secret injection, for example
`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
enabled; this path is ignored as operational secret material.
## Known state caveat
kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
internal `kube_system_secrets` handling. This task does not claim that risk is
solved. Task 5 must verify the generated plan and state before production apply
and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
keys, and decrypted secrets are absent from committed files and unsafe state
evidence.
+16
View File
@@ -0,0 +1,16 @@
terraform {
backend "s3" {
bucket = "gitea-runner-hectic-lab"
key = "gitea-runners/kube-hetzner/terraform.tfstate"
region = "fsn1"
encrypt = true
use_lockfile = true
}
}
check "remote_state_contract" {
assert {
condition = local.production_remote_state
error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
}
}
+60
View File
@@ -0,0 +1,60 @@
locals {
default_storage_class = "hcloud-volumes"
control_plane_nodepools = [
{
name = "control-plane"
server_type = var.control_plane_server_type
location = var.hetzner_location
labels = []
taints = []
count = 1
},
]
agent_nodepools = [
{
name = "runner-workers"
server_type = var.worker_server_type
location = var.hetzner_location
labels = ["node-role.hectic-lab/gitea-runner=true"]
taints = []
count = var.worker_count
},
]
}
module "kube_hetzner" {
source = "kube-hetzner/kube-hetzner/hcloud"
version = "2.19.3"
providers = {
hcloud = hcloud
}
hcloud_token = var.hcloud_token
ssh_public_key = var.ssh_public_key
ssh_private_key = var.ssh_private_key
cluster_name = var.cluster_name
base_domain = var.base_domain
# kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
# expose that expected path without outputting kubeconfig private key material.
create_kubeconfig = true
network_region = var.network_region
load_balancer_location = var.hetzner_location
control_plane_nodepools = local.control_plane_nodepools
agent_nodepools = local.agent_nodepools
# Hetzner CSI is the required StorageClass provider for runner PVCs.
disable_hetzner_csi = false
# Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
enable_longhorn = false
# Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
# raise worker_count to 5 or increase worker_server_type after validating pod
# CPU, memory, and ephemeral-storage pressure in Task 11.
}
+22
View File
@@ -0,0 +1,22 @@
output "kubeconfig_path" {
description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
}
output "cluster_name" {
description = "kube-hetzner cluster name."
value = var.cluster_name
}
output "node_pool_names" {
description = "Control-plane and worker node pool names used by this stack."
value = {
control_plane = [for pool in local.control_plane_nodepools : pool.name]
workers = [for pool in local.agent_nodepools : pool.name]
}
}
output "default_storage_class" {
description = "Default Hetzner CSI StorageClass expected for runner PVCs."
value = local.default_storage_class
}
@@ -0,0 +1 @@
provider "hcloud" {}
+74
View File
@@ -0,0 +1,74 @@
variable "hcloud_token" {
description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
type = string
sensitive = true
}
variable "ssh_public_key" {
description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
type = string
}
variable "ssh_private_key" {
description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
type = string
sensitive = true
}
variable "cluster_name" {
description = "Name for the kube-hetzner runner cluster."
type = string
default = "gitea-runners"
validation {
condition = can(regex("^[a-z0-9-]+$", var.cluster_name))
error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
}
}
variable "hetzner_location" {
description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
type = string
default = "fsn1"
}
variable "network_region" {
description = "Hetzner private network region. eu-central covers fsn1."
type = string
default = "eu-central"
}
variable "control_plane_server_type" {
description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
type = string
default = "cpx21"
}
variable "worker_server_type" {
description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
type = string
default = "cpx31"
}
variable "worker_count" {
description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
type = number
default = 3
validation {
condition = var.worker_count >= 1
error_message = "worker_count must be at least 1."
}
}
variable "kubeconfig_path" {
description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
type = string
default = null
}
variable "base_domain" {
description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
type = string
default = ""
}
+17
View File
@@ -0,0 +1,17 @@
terraform {
required_version = ">= 1.10.1"
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = "1.60.1"
}
}
}
locals {
kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud"
kube_hetzner_module_version = "2.19.3"
hcloud_provider_minimum = ">= 1.59.0"
production_remote_state = true
}
+503
View File
@@ -0,0 +1,503 @@
# Gitea Runner Infrastructure Runbook
## Scope
This directory is the repo-owned boundary for the first Gitea Actions runner
pool. Task 1 only establishes the scaffold and immutable decision contract;
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
and a Nix-capable runner image under the existing subdirectories.
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
workflow job containers are ephemeral, while each runner pod keeps its runner
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
## Immutable decisions
- Infrastructure is managed with OpenTofu command examples only, using the
`tofu` CLI.
- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
- Runner implementation is the non-Enterprise `gitea/runner`.
- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
environment variables are not the contract.
- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
`/data`, including `/data/.runner`.
- Container builds run through privileged rootful DinD inside trusted runner
pods; host Docker socket mounting is not an implementation path.
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
the Nix-capable image has been pushed and a concrete registry-reported digest
is added to the runner ConfigMap.
- First scope is trusted internal workflows only, with no untrusted fork or PR
workflow support.
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
## Lifecycle boundaries
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
contract, Hetzner provider configuration, and kube-hetzner module wiring.
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
handoff; package or flake output changes are outside Task 1.
- `infra/gitea-runners/runbook.md`: this contract plus later operational
commands, rollback notes, and acceptance evidence references.
## Guardrails
- Enterprise ARC/actions-runner-controller are rejected alternatives and must
not be implemented here. Do not add ARC custom resources, controller install
instructions, or GitHub Actions ARC assumptions.
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
acceptable for trusted internal jobs.
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
runner pool.
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
S3 credentials, decrypted SOPS files, or SOPS age keys.
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
runner token; Kubernetes receives the token as a mounted file secret instead.
- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
jobs must reach the public HTTPS service.
## Initial acceptance commands
Run from the repository root:
```sh
test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
test -f infra/gitea-runners/runbook.md
grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
grep -R "[t]erraform " infra/gitea-runners || true
grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
```
Expected outcomes: the directory and file checks exit 0; the architecture-term
grep shows this contract; ARC references appear only in the rejected-alternative
guardrail above; there are no forbidden CLI command examples and no unresolved
decision placeholders.
## Downstream placeholders
- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
- Task 3: add SOPS secret contract and runtime token delivery details.
- Task 4: define or package the Nix-capable runner image for the `nix` label.
- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
and document cleanup, rollback, and scaling operations.
## Runner lifecycle cleanup
All lifecycle commands are scoped to the runner namespace:
```sh
kubectl -n gitea-runners get statefulset gitea-runner
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
```
The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
does not delete pods, PVCs, Docker data, or Gitea runner registrations.
Run the same inventory on demand without waiting for the schedule:
```sh
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
```
The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
operator to remove finished jobs manually.
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
pod loses `/data/.runner`. That runner identity must then be deregistered from
Gitea or the replacement pod must be allowed to re-register intentionally with
the current organization runner token. Do not delete an active runner PVC as a
normal cleanup step.
Non-UI Gitea registration reconciliation uses the Gitea API with a separate
admin token. Store that token outside this repository and pass it as a file; do
not print it:
```sh
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
--restart=Never \
--image=curlimages/curl:8.10.1 \
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
```
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
has completed and its logs have been collected. Do not keep this admin token in
the runner namespace longer than the reconciliation window.
Only remove a stale Gitea runner registration after the corresponding pod/PVC
was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
cleanup is a fallback, not the only path. Record the removed runner name and the
Kubernetes PVC/pod deletion that made it stale.
After the dry-run list identifies a stale registration and the PVC/pod deletion
has been recorded, remove that exact Gitea runner by id through the API:
```sh
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
umask 077
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
trap 'rm -f "$curl_config"' EXIT
{
printf 'request = "DELETE"\n'
printf 'header = "Authorization: token '
cat /secure/path/gitea-admin-token
printf '"\n'
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
} > "$curl_config"
curl -fsS --config "$curl_config"
```
Do not run the delete command for a runner that still has an active
`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
being intentionally reset for re-registration.
## Docker-in-Docker storage cleanup
Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
list disk usage before pruning, and run the command only against the `docker`
container in runner pods in `gitea-runners`. Because this storage is pod-local,
loop over pods for pool-wide cleanup:
```sh
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
```
For one pod, replace the StatefulSet target with the pod name:
```sh
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
```
Do not run host-level Docker cleanup commands and do not mount or prune a host
Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
Kubernetes; the `/data` PVC remains and still controls runner identity.
## Token rotation
Rotate the Gitea organization runner token without printing decrypted values:
```sh
sops sus/gitea-runners.yaml
umask 077
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
trap 'rm -f "$token_file"' EXIT
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
kubectl -n gitea-runners create secret generic gitea-runner-token \
--from-file=token="$token_file" \
--dry-run=client \
-o yaml | kubectl -n gitea-runners apply -f -
kubectl -n gitea-runners rollout restart statefulset/gitea-runner
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
```
The `rollout restart` command above is the controlled restart path for this
StatefulSet. Observe the rollout and each ordinal until all replacement pods are
Ready; do not delete runner pods directly as part of normal token rotation:
```sh
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
```
Verification must confirm the token file mount remains present while the token
value never appears in logs or evidence:
```sh
kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
```
## Deploy and status
These commands are executable only when the external inputs are available:
- `TF_VAR_hcloud_token`
- `TF_VAR_ssh_public_key`
- `TF_VAR_ssh_private_key`
- S3 backend credentials and endpoint access
- a matching SOPS age identity for `sus/gitea-runners.yaml`
- `kubectl` access to the target cluster
- a concrete digest for the pushed Nix-capable runner image, if enabling the
`nix` label
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
stale kubeconfig files.
Before production Kubernetes apply or rollout, satisfy both manifest gates:
1. Create or update the `gitea-runner-token` Secret from SOPS. The active
Kustomize overlay intentionally does not include a placeholder Secret, but
the StatefulSet still mounts `secretName: gitea-runner-token` as
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
image has been pushed successfully. Enable the `nix` label only by adding a
digest-pinned `docker://` mapping with the exact registry-reported sha256
digest from that push.
Use the same SOPS materialization pattern as token rotation before applying the
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
target namespace; the full overlay remains gated on the Secret and digest
decisions:
```sh
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
umask 077
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
trap 'rm -f "$token_file"' EXIT
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
kubectl -n gitea-runners create secret generic gitea-runner-token \
--from-file=token="$token_file" \
--dry-run=client \
-o yaml | kubectl -n gitea-runners apply -f -
```
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
above succeeds. Do not claim or enable the `nix` runner label until the image
publication step has produced the concrete digest.
```sh
tofu -chdir=infra/gitea-runners/opentofu init
tofu -chdir=infra/gitea-runners/opentofu validate
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
kubectl config current-context
kubectl get nodes -o wide
kubectl get sc
kubectl apply -k infra/gitea-runners/k8s
kubectl -n gitea-runners get statefulset gitea-runner
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
```
Expected status after deploy:
- `kubectl config current-context` names the runner cluster context.
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
## Scale 5 to 10 to 5
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
```sh
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
# Run the concurrent smoke workflows now.
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
```
After scaling back down, inspect the cleanup dry-run and deregister any stale
runner registrations only for pods or PVCs that were intentionally removed.
## Cleanup and stale runner deregistration
Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
PVC candidates whose pod is gone. It must not delete active resources.
```sh
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
```
Pool-wide DinD storage checks and cleanup:
```sh
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
done
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
done
```
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
replacement pod re-register intentionally with the current organization token.
Never delete an active runner PVC as routine cleanup.
Non-UI Gitea registration reconciliation uses an admin token stored outside this
repository:
```sh
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
--restart=Never \
--image=curlimages/curl:8.10.1 \
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
```
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
pod has completed and its logs have been collected.
After the dry-run list identifies a stale registration and the PVC or pod
deletion has been recorded, remove that exact Gitea runner by id through the
API:
```sh
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
umask 077
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
trap 'rm -f "$curl_config"' EXIT
{
printf 'request = "DELETE"\n'
printf 'header = "Authorization: token '
cat /secure/path/gitea-admin-token
printf '"\n'
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
} > "$curl_config"
curl -fsS --config "$curl_config"
```
Do not run the delete command for a runner that still has an active
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
being intentionally reset for re-registration.
## Application rollback
Rollback the app layer only. Do not use this section to destroy the cluster.
```sh
kubectl -n gitea-runners rollout history statefulset/gitea-runner
kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
```
If a manifest rollback is needed, reapply the repo overlay after checking out the
known-good revision, then re-run the rollout checks:
```sh
kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
```
## Full cluster teardown
This destroys Hetzner resources owned by the kube-hetzner stack, including the
`gitea-runners` cluster nodes, the `control-plane` node pool, the
`runner-workers` node pool, the cluster network, load balancer resources,
firewall objects, and any attached Hetzner CSI volumes still managed by the
stack. Do not run teardown unless the destruction is intentional.
```sh
tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
```
## Partial OpenTofu apply recovery
If `tofu apply` fails after creating some resources, do not destroy blindly.
First reconcile state and inspect what the stack thinks exists:
```sh
tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
tofu -chdir=infra/gitea-runners/opentofu state list
```
Then rerun the normal plan path. Use `-target` only as a last resort when a
single resource is stuck and the drift is understood.
## S3 backend recovery
If backend init or state access fails, first verify the bucket and versioning
outside OpenTofu, then reconfigure the backend:
```sh
nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
tofu -chdir=infra/gitea-runners/opentofu plan
```
If the backend reports a stale lock, confirm no `tofu` process is active, then
use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
unlock a live plan or apply.
## Gitea outage troubleshooting
Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
containers.
```sh
kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
```
If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
`/data/.runner` just because the service is unavailable. Once Gitea returns,
repeat the token rotation or re-registration path if a pod restarted while the
service was unavailable and lost its runner identity.
## Release checklist
Do not release unless the following evidence files exist and are readable:
- `.sisyphus/evidence/task-5-cluster-plan.txt`
- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
- `.sisyphus/evidence/task-9-deploy.txt`
- `.sisyphus/evidence/task-9-secret-mount.txt`
- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
- `.sisyphus/evidence/task-10-nix-workflow.txt`
- `.sisyphus/evidence/task-11-scale.txt`
- `.sisyphus/evidence/task-11-restart-cleanup.txt`
If any evidence file is missing, stop and collect it before treating the runbook
as complete.
+71 -12
View File
@@ -26,12 +26,39 @@
"aarch64-darwin" "aarch64-darwin"
]; ];
cudaUnfreeNames = [
"cuda_nvcc"
"cuda_cudart"
"cuda_cuobjdump"
"cuda_cupti"
"cuda_nvdisasm"
"cuda_cccl"
"cuda_nvml_dev"
"cuda_nvrtc"
"cuda_nvtx"
"cuda_profiler_api"
"libcusparse_lt"
"libcublas"
"libcufft"
"libcufile"
"libcurand"
"libcusolver"
"libnvjitlink"
"libcusparse"
"cudnn"
];
cudaUnfreePredicate = pkg:
builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
forSystemsWithPkgs = supportedSystems: pkgOverlays: f: forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
builtins.foldl' ( builtins.foldl' (
acc: system: let acc: system: let
pkgs = import nixpkgs { pkgs = import nixpkgs {
inherit system; inherit system;
overlays = pkgOverlays; overlays = pkgOverlays;
config.allowUnfreePredicate = cudaUnfreePredicate;
}; };
systemOutputs = f { systemOutputs = f {
system = system; system = system;
@@ -58,7 +85,7 @@
else {}; else {};
in { in {
# -- For all systems -- # -- For all systems --
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems; inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
forSystems = systems: nixpkgs.lib.genAttrs systems; forSystems = systems: nixpkgs.lib.genAttrs systems;
forAllSystems = nixpkgs.lib.genAttrs AllSystems; forAllSystems = nixpkgs.lib.genAttrs AllSystems;
@@ -67,6 +94,7 @@ in {
logs = builtins.readFile ./shell/logs.sh; logs = builtins.readFile ./shell/logs.sh;
check-tool = builtins.readFile ./shell/check-tool.sh; check-tool = builtins.readFile ./shell/check-tool.sh;
local-dir = builtins.readFile ./shell/local-dir.sh; local-dir = builtins.readFile ./shell/local-dir.sh;
load-sops = builtins.readFile ./shell/load-sops.sh;
}; };
sharedShellAliases = { sharedShellAliases = {
@@ -102,7 +130,7 @@ in {
# Consolidated SQL bundles for the `hectic` schema. Single source of truth # Consolidated SQL bundles for the `hectic` schema. Single source of truth
# for everything that creates objects in the `hectic` namespace, used by # for everything that creates objects in the `hectic` namespace, used by
# migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply # migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
# the full bundle via lib/hook/apply-hectic-bundle.sh. # the full bundle via lib/hook/apply-hectic-bundle.sh.
# #
# The whole hectic system shares one `versionString`; `hectic-version.sql` # The whole hectic system shares one `versionString`; `hectic-version.sql`
@@ -115,19 +143,42 @@ in {
hectic = let hectic = let
versionString = lib.fileContents ./hook/sql/HECTIC_VERSION; versionString = lib.fileContents ./hook/sql/HECTIC_VERSION;
static = path: { inherit path; sql = builtins.readFile path; }; static = path: { inherit path; sql = builtins.readFile path; };
templated = path: { templated = path: let
sql = builtins.replaceStrings sql = builtins.replaceStrings
[ "@HECTIC_VERSION@" ] [ "@HECTIC_VERSION@" ]
[ versionString ] [ versionString ]
(builtins.readFile path); (builtins.readFile path);
in {
inherit sql;
path = builtins.toFile (builtins.baseNameOf (toString path)) sql;
}; };
in { in rec {
inherit versionString; inherit versionString;
version = templated ./hook/sql/hectic-version.sql; version = templated ./hook/sql/hectic-version.sql;
secret = static ./hook/sql/hectic-secret.sql; secret = static ./hook/sql/hectic-secret.sql;
migration = static ./hook/sql/hectic-migration.sql; migration = static ./hook/sql/hectic-migration.sql;
inheritance = static ./hook/sql/hectic-inheritance.sql; inheritance = static ./hook/sql/hectic-inheritance.sql;
applyBundleScript = ./hook/apply-hectic-bundle.sh; bundleFiles = [
version.path
secret.path
migration.path
inheritance.path
];
applyBundleScript =
builtins.replaceStrings
[
"@HECTIC_VERSION_SQL@"
"@HECTIC_SECRET_SQL@"
"@HECTIC_MIGRATION_SQL@"
"@HECTIC_INHERITANCE_SQL@"
]
[
"${version.path}"
"${secret.path}"
"${migration.path}"
"${inheritance.path}"
]
(builtins.readFile ./hook/apply-hectic-bundle.sh);
}; };
# Back-compat alias. Prefer `self.lib.hectic.inheritance`. # Back-compat alias. Prefer `self.lib.hectic.inheritance`.
@@ -164,19 +215,27 @@ in {
readModulesRecursive' = path: extraArgs: readModulesRecursive' = path: extraArgs:
with lib; with lib;
with builtins; let with builtins; let
paths = pipe "${path}" [ collectPaths = dir: prefix:
(filesystem.listFilesRecursive) concatLists (mapAttrsToList (name: type: let
(filter (hasSuffix ".nix")) path' = dir + "/${name}";
]; name' = if prefix == "" then name else "${prefix}/${name}";
in
if type == "directory"
then collectPaths path' name'
else [{
inherit path';
name = name';
}]
) (readDir dir));
paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
pathToName = flip pipe [ pathToName = flip pipe [
(removePrefix "${path}/")
(replaceStrings ["/" ".nix"] ["." ""]) (replaceStrings ["/" ".nix"] ["." ""])
(removeSuffix ".nix") (removeSuffix ".nix")
]; ];
attrList = attrList =
map (path': { map (path': {
name = pathToName (unsafeDiscardStringContext path'); name = pathToName path'.name;
value = import path' extraArgs; value = import path'.path' extraArgs;
}) })
paths; paths;
in in
+17 -24
View File
@@ -7,17 +7,12 @@
# #
# Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE. # Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
# #
# Required env (caller injects from Nix):
# HECTIC_VERSION_SQL - path to hectic-version.sql (substituted)
# HECTIC_SECRET_SQL - path to hectic-secret.sql
# HECTIC_MIGRATION_SQL - path to hectic-migration.sql
# HECTIC_INHERITANCE_SQL - path to hectic-inheritance.sql
#
# Usage: # Usage:
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>] # apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
# #
# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via # If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
# hectic.load_secrets_from_env() after the bundle is applied. # hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
# SQL file paths are substituted by Nix evaluation time.
apply_hectic_bundle() { apply_hectic_bundle() {
pgurl="${1:-}" pgurl="${1:-}"
@@ -28,29 +23,27 @@ apply_hectic_bundle() {
return 3 return 3
fi fi
for var in HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL; do set -- \
eval "val=\${$var:-}" "@HECTIC_VERSION_SQL@" \
if [ -z "$val" ]; then "@HECTIC_SECRET_SQL@" \
printf '%s\n' "apply-hectic-bundle: $var not set" >&2 "@HECTIC_MIGRATION_SQL@" \
return 3 "@HECTIC_INHERITANCE_SQL@"
fi
if [ ! -r "$val" ]; then for sql_path do
printf '%s\n' "apply-hectic-bundle: $var not readable: $val" >&2 if [ ! -r "$sql_path" ]; then
printf '%s\n' "apply-hectic-bundle: SQL file not readable: $sql_path" >&2
return 1 return 1
fi fi
done done
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_VERSION_SQL" || return 1 for sql_path do
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_SECRET_SQL" || return 1 psql "$pgurl" -v ON_ERROR_STOP=1 -f "$sql_path" || return 1
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_MIGRATION_SQL" || return 1 done
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_INHERITANCE_SQL" || return 1
if [ -n "$env_content" ]; then if [ -n "$env_content" ]; then
# Dollar-quote with $ps_env$ tag to preserve all content verbatim. env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1 psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
SELECT hectic.load_secrets_from_env(\$ps_env\$ SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
$env_content
\$ps_env\$);
SQL SQL
fi fi
+23 -25
View File
@@ -4,8 +4,8 @@ Single source of truth for every object created in the `hectic` PostgreSQL
schema. Consumed by: schema. Consumed by:
- `package/migrator` — applies the bundle on `migrator init` (mandatory). - `package/migrator` — applies the bundle on `migrator init` (mandatory).
- `package/db-tool` — applies the bundle in `database hydrate` (default; opt - `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
out with `--no-hook`). (default; opt out with `--no-hook`) and in `db-ops secrets load`.
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the - External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
paths exposed via `self.lib.hectic.*.path`. paths exposed via `self.lib.hectic.*.path`.
@@ -42,23 +42,26 @@ that already matches.
```nix ```nix
self.lib.hectic = { self.lib.hectic = {
versionString; # e.g. "0.1.0" versionString; # e.g. "0.1.0"
version = { sql; }; # templated version = { sql; path; }; # templated
secret = { sql; path; }; secret = { sql; path; };
migration = { sql; path; }; migration = { sql; path; };
inheritance = { sql; path; }; inheritance = { sql; path; };
applyBundleScript; # ./hook/apply-hectic-bundle.sh bundleFiles; # ordered bundle file paths
applyBundleScript; # generated helper shell source with paths embedded
}; };
``` ```
`.sql` is the file contents as a string. `.path` is the Nix store path of the `.sql` is the file contents as a string. `.path` is the Nix store path of the
verbatim source (only available on non-templated entries; consumers needing a materialized file to pass to `psql -f`. `version.path` is generated at Nix
materialized version of `version.sql` must do evaluation time from the templated SQL; the other `*.path` entries point at the
`pkgs.runCommand "hectic-version.sql" { text = self.lib.hectic.version.sql; passAsFile = ["text"]; } ''cp "$textPath" "$out"''`). verbatim source files in the store.
## Shell helper (`apply-hectic-bundle.sh`) ## Shell helper (`apply-hectic-bundle.sh`)
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper sourced by both `lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
`migrator` and `db-tool`. Public entry point: `self.lib.hectic.applyBundleScript` is the generated shell source with concrete
SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
shell source directly into their generated scripts. Public entry point:
```sh ```sh
apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>] apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
@@ -70,28 +73,23 @@ apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
dollar-quoted (`$ps_env$`) string so secret values cannot terminate the dollar-quoted (`$ps_env$`) string so secret values cannot terminate the
literal. literal.
Required environment (paths to the SQL files): The SQL file paths are embedded into the helper at Nix evaluation time, so
callers only need to source the generated script and call the function.
- `HECTIC_VERSION_SQL` External consumers that do not want to source the helper can still invoke
- `HECTIC_SECRET_SQL` `psql -f` against `self.lib.hectic.bundleFiles` or the individual
- `HECTIC_MIGRATION_SQL` `self.lib.hectic.*.path` entries directly.
- `HECTIC_INHERITANCE_SQL`
`migrator` and `db-tool` set these via Nix at build time. External consumers
typically invoke `psql -f` against the paths directly instead of sourcing the
helper.
## Adding a new SQL file ## Adding a new SQL file
1. Add `lib/hook/sql/hectic-<name>.sql`. 1. Add `lib/hook/sql/hectic-<name>.sql`.
2. Wire it into `lib/default.nix` under `lib.hectic.<name>`. 2. Wire it into `lib/default.nix` under `lib.hectic.<name>`.
3. Inject `HECTIC_<NAME>_SQL` in both `package/migrator/default.nix` and 3. Add its `.path` to `lib.hectic.bundleFiles` in the correct order.
`package/db-tool/default.nix`. 4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
4. Append a `psql -f "$HECTIC_<NAME>_SQL"` step to update `lib/hook/apply-hectic-bundle.sh` to apply the file.
`lib/hook/apply-hectic-bundle.sh` in the correct order.
5. Bump `HECTIC_VERSION` if the new content changes existing semantics. 5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/` 6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
and `test/package/db-tool/test/postgresql/hydrate-hook/`. `test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
bundle-loading coverage.
## Versioning ## Versioning
+204
View File
@@ -0,0 +1,204 @@
load_sops_shell_quote() {
printf "'"
printf '%s' "$1" | sed "s/'/'\"'\"'/g"
printf "'"
}
load_sops_normalize_key() {
load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
case "$load_sops_normalized_key" in
''|[!A-Z_]*|*[!A-Z0-9_]*)
printf 'load-sops: invalid environment name after key normalization\n' >&2
return 1
;;
esac
printf '%s\n' "$load_sops_normalized_key"
}
load_sops_env_from_sops_file() {
load_sops_file=$1
load_sops_extract=${2-}
if ! command -v sops >/dev/null 2>&1; then
printf 'load-sops: required tool `sops` not found\n' >&2
return 1
fi
if ! command -v yq >/dev/null 2>&1; then
printf 'load-sops: required tool `yq` not found\n' >&2
return 1
fi
load_sops_decrypted=''
load_sops_attempt=0
load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
while :; do
if [ -n "$load_sops_extract" ]; then
if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
load_sops_status=0
else
load_sops_status=$?
fi
else
if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
load_sops_status=0
else
load_sops_status=$?
fi
fi
if [ "$load_sops_status" -eq 0 ]; then
break
fi
if [ "$load_sops_prompt" != 1 ]; then
printf 'load-sops: failed to decrypt file\n' >&2
return 1
fi
if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
return 1
fi
load_sops_attempt=$((load_sops_attempt + 1))
if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
printf 'load-sops: decrypt failed after configured retries\n' >&2
return 1
fi
load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
case "$load_sops_use_script" in
auto)
if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
load_sops_script_status=0
else
load_sops_script_status=$?
fi
if [ "$load_sops_script_status" -eq 0 ]; then
continue
fi
fi
;;
1)
if ! command -v script >/dev/null 2>&1; then
printf 'load-sops: required tool `script` not found\n' >&2
return 1
fi
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
load_sops_script_status=0
else
load_sops_script_status=$?
fi
if [ "$load_sops_script_status" -eq 0 ]; then
continue
fi
;;
0)
;;
*)
printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
return 1
;;
esac
printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
printf 'load-sops: failed to read prompt input\n' >&2
return 1
fi
export SOPS_AGE_KEY_CMD
done
load_sops_env_from_yaml_text "$load_sops_decrypted"
}
load_sops_env_from_yaml_file() {
load_sops_file=$1
if ! command -v yq >/dev/null 2>&1; then
printf 'load-sops: required tool `yq` not found\n' >&2
return 1
fi
load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
}
load_sops_env_from_yaml_text() {
load_sops_yaml=$1
load_sops_seen=''
if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
load_sops_keys_status=0
else
load_sops_keys_status=$?
fi
if [ "$load_sops_keys_status" -ne 0 ]; then
printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
return 1
fi
while IFS= read -r load_sops_key; do
[ -n "$load_sops_key" ] || continue
load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
case "
$load_sops_seen
" in
*"
$load_sops_name
"*)
if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
printf 'load-sops: normalized environment name collision\n' >&2
return 1
fi
;;
esac
load_sops_seen=${load_sops_seen}${load_sops_seen:+"
"}$load_sops_name
load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
printf 'load-sops: failed to inspect YAML value kind\n' >&2
return 1
}
load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
printf 'load-sops: failed to inspect YAML value tag\n' >&2
return 1
}
if [ "$load_sops_kind" != scalar ]; then
printf 'load-sops: top-level YAML values must be scalars\n' >&2
return 1
fi
if [ "$load_sops_tag" = '!!null' ]; then
printf 'load-sops: top-level YAML null values are not supported\n' >&2
return 1
fi
if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
if [ -n "$load_sops_already_set" ]; then
continue
fi
fi
load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
printf 'load-sops: failed to read YAML scalar value\n' >&2
return 1
}
load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
eval "export $load_sops_name=$load_sops_quoted"
done <<EOF
$load_sops_keys
EOF
}
+2 -2
View File
@@ -9,11 +9,11 @@ with self.lib;
let let
# Combine hectic modules into one # Combine hectic modules into one
hectic.imports = attrValues ( hectic.imports = attrValues (
readModulesRecursive' ./hectic { inherit flake self inputs; } readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
); );
# Read generic modules separately # Read generic modules separately
generic = readModulesRecursive' generic = readModulesRecursive'
./generic (flake + "/nixos/module/generic")
{ inherit flake self inputs; }; { inherit flake self inputs; };
in generic // { in generic // {
inherit hectic; inherit hectic;
@@ -18,6 +18,7 @@
]; ];
adminNames = [ "yukkop" ]; adminNames = [ "yukkop" ];
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
cfg = config.hectic.generic.matrix-cluster; cfg = config.hectic.generic.matrix-cluster;
in { in {
@@ -39,7 +40,7 @@ in {
value = { value = {
key = "matrix/users/${name}/password"; key = "matrix/users/${name}/password";
owner = "matrix-synapse"; owner = "matrix-synapse";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
}) userNames }) userNames
); );
+7
View File
@@ -250,6 +250,7 @@ in {
# failover flip does not need a separate provisioning step. # failover flip does not need a separate provisioning step.
systemd.tmpfiles.rules = [ systemd.tmpfiles.rules = [
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -" "d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
"Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
]; ];
systemd.services.matrix-cluster-signing-key = { systemd.services.matrix-cluster-signing-key = {
@@ -319,6 +320,12 @@ in {
media_store_path = s3Cfg.mediaStorePath; media_store_path = s3Cfg.mediaStorePath;
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key"; signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
# Tolerate bursty Element/iPhone presence syncs without disabling limits.
rc_presence.per_user = {
per_second = 0.5;
burst_count = 5;
};
experimental_features = { experimental_features = {
msc3266_enabled = true; msc3266_enabled = true;
msc4140_enabled = true; msc4140_enabled = true;
+10 -2
View File
@@ -1,7 +1,7 @@
{ {
inputs, inputs,
flake,
self, self,
...
}: { }: {
pkgs, pkgs,
lib, lib,
@@ -27,7 +27,15 @@ in {
users.defaultUserShell = pkgs.zsh; users.defaultUserShell = pkgs.zsh;
# Enable flakes and new 'nix' command # Enable flakes and new 'nix' command
nix.settings.experimental-features = "nix-command flakes"; nix.settings = {
experimental-features = "nix-command flakes";
extra-substituters = [
"https://cache.hectic-lab.com/hectic"
];
extra-trusted-public-keys = [
"hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
];
};
networking.firewall.enable = true; networking.firewall.enable = true;
+52 -10
View File
@@ -1,7 +1,5 @@
{ {
inputs, ...
flake,
self,
}: }:
{ {
pkgs, pkgs,
@@ -11,6 +9,13 @@
}: let }: let
cfg = config.hectic.hardware.hetzner-cloud; cfg = config.hectic.hardware.hetzner-cloud;
isNewer = cfg.generation == "newer"; isNewer = cfg.generation == "newer";
networkMatchConfig =
(lib.optionalAttrs (cfg.networkMatchConfigName != null) {
Name = cfg.networkMatchConfigName;
})
// (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
PermanentMACAddress = cfg.networkMatchConfigMac;
});
in { in {
options.hectic.hardware.hetzner-cloud = { options.hectic.hardware.hetzner-cloud = {
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations"; enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
@@ -51,6 +56,14 @@ in {
''; '';
}; };
floatingIpv4 = lib.mkOption {
type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
default = null;
example = "188.243.124.247";
description = ''
Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
'';
};
device = lib.mkOption { device = lib.mkOption {
type = lib.types.str; type = lib.types.str;
default = if isNewer then "/dev/nvme0n1" else "/dev/sda"; default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
@@ -65,14 +78,27 @@ in {
''; '';
}; };
networkMatchConfigName = lib.mkOption { networkMatchConfigName = lib.mkOption {
type = lib.types.strMatching "^(enp1s0|ens3|eth0)$"; type = with lib.types; nullOr str;
default = null;
example = "enp1s0"; example = "enp1s0";
description = '' description = ''
type of network conection, Optional interface name to match in systemd-networkd.
on older hetzner servers may be `ens3` or else
on newer probably `enp1s0`
you can use `networkctl list` on server to know it Prefer `networkMatchConfigMac` for stable matching across rescue
images and installed systems that may rename interfaces differently.
You can use `networkctl list` on server to know it.
'';
};
networkMatchConfigMac = lib.mkOption {
type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
default = null;
example = "92:00:08:4a:b0:32";
description = ''
Optional permanent MAC address to match in systemd-networkd.
This is the preferred Hetzner Cloud matching method because interface
names can differ between rescue images and installed NixOS systems.
''; '';
}; };
}; };
@@ -80,6 +106,15 @@ in {
config = lib.mkIf cfg.enable (lib.mkMerge config = lib.mkIf cfg.enable (lib.mkMerge
[ [
{ {
boot.loader.systemd-boot.enable = false;
boot.loader.efi.canTouchEfiVariables = false;
boot.loader.grub = {
enable = true;
efiSupport = true;
efiInstallAsRemovable = true;
device = "nodev";
};
boot.initrd.availableKernelModules = [ boot.initrd.availableKernelModules = [
"ata_piix" "ata_piix"
"uhci_hcd" "uhci_hcd"
@@ -90,12 +125,12 @@ in {
networking.useNetworkd = true; networking.useNetworkd = true;
systemd.network.enable = true; systemd.network.enable = true;
systemd.network.networks."30-wan" = { systemd.network.networks."30-wan" = {
matchConfig.Name = cfg.networkMatchConfigName; matchConfig = networkMatchConfig;
networkConfig.DHCP = "no"; networkConfig.DHCP = "no";
address = [ address = [
"${cfg.ipv4}/32" "${cfg.ipv4}/32"
"${cfg.ipv6}::/64" "${cfg.ipv6}::/64"
]; ] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
routes = [ routes = [
{ Gateway = "172.31.1.1"; GatewayOnLink = true; } { Gateway = "172.31.1.1"; GatewayOnLink = true; }
{ Gateway = "fe80::1"; } { Gateway = "fe80::1"; }
@@ -137,6 +172,13 @@ in {
}; };
}; };
}; };
assertions = [
{
assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
}
];
} }
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") { (lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
boot.initrd.kernelModules = [ "virtio_gpu" ]; boot.initrd.kernelModules = [ "virtio_gpu" ];
+134
View File
@@ -0,0 +1,134 @@
{ ... }:
{
lib,
config,
...
}: let
cfg = config.hectic.hardware.njalla;
in {
options.hectic.hardware.njalla = {
enable = lib.mkEnableOption "Enable njalla hardware configurations";
ipv4 = lib.mkOption {
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
example = "185.193.126.103";
description = ''
Njalla IPv4 address assigned to the host.
'';
};
ipv4PrefixLength = lib.mkOption {
type = lib.types.int;
default = 24;
example = 24;
description = ''
Njalla IPv4 prefix length.
'';
};
ipv4Gateway = lib.mkOption {
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
default = "185.193.126.1";
example = "185.193.126.1";
description = ''
Njalla IPv4 gateway.
'';
};
ipv6 = lib.mkOption {
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
example = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
description = ''
Njalla IPv6 address assigned to the host.
'';
};
ipv6PrefixLength = lib.mkOption {
type = lib.types.int;
default = 64;
example = 64;
description = ''
Njalla IPv6 prefix length.
'';
};
ipv6Gateway = lib.mkOption {
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
default = "2a0a:3840:1337:126::1";
example = "2a0a:3840:1337:126::1";
description = ''
Njalla IPv6 gateway.
'';
};
networkMatchConfigName = lib.mkOption {
type = lib.types.str;
default = "eth0";
example = "eth0";
description = ''
Njalla container network interface name.
'';
};
device = lib.mkOption {
type = lib.types.str;
default = "/dev/vda";
example = "/dev/disk/by-id/virtio-root";
description = ''
Njalla installation disk for disko/nixos-anywhere.
`/dev/vda` is the default block device visible on the inspected Njalla
host. Prefer a stable `/dev/disk/by-id/...` path when available.
'';
};
enableDisko = lib.mkOption {
type = lib.types.bool;
default = true;
description = ''
Whether to provide a disko layout for nixos-anywhere installs.
'';
};
};
config = lib.mkIf cfg.enable (lib.mkMerge [
{
boot.isContainer = true;
networking.useDHCP = false;
networking.useNetworkd = true;
systemd.network.enable = true;
systemd.network.networks."30-wan" = {
matchConfig.Name = cfg.networkMatchConfigName;
networkConfig.DHCP = "no";
address = [
"${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
"${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
];
routes = [
{ Gateway = cfg.ipv4Gateway; }
{ Gateway = cfg.ipv6Gateway; }
];
};
networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
}
(lib.mkIf cfg.enableDisko {
boot.loader.grub.device = cfg.device;
disko.devices.disk.main = {
type = "disk";
device = cfg.device;
content = {
type = "gpt";
partitions = {
boot = {
size = "1M";
type = "EF02";
priority = 1;
};
root = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
};
};
};
};
})
]);
}
+1 -30
View File
@@ -25,36 +25,7 @@ in {
programs.bash.shellAliases.tmux = "tmux a"; programs.bash.shellAliases.tmux = "tmux a";
home-manager.sharedModules = [ home-manager.sharedModules = [
{ (flake + "/home/module/program/tmux.nix")
programs.tmux = {
enable = true;
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
keyMode = "vi";
escapeTime = 500;
historyLimit = 50000;
newSession = true;
extraConfig = ''
# resurrect
set -g @resurrect-strategy-vim 'session'
set -g @resurrect-strategy-nvim 'session'
set -g @resurrect-capture-pane-contents 'on'
resurrect_dir="$HOME/.tmux/resurrect"
set -g @resurrect-dir $resurrect_dir
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
# continuum
set -g @continuum-restore 'on'
set -g @continuum-boot 'on'
set -g @continuum-save-interval '10'
bind-key -T copy-mode-vi v send-keys -X begin-selection
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
bind-key O select-pane -t :.-
'';
};
}
]; ];
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05"; home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
+73
View File
@@ -0,0 +1,73 @@
{ ... }: {
lib,
config,
...
}: let
cfg = config.hectic.services.attic;
in {
options.hectic.services.attic = {
enable = lib.mkEnableOption "Attic binary cache server";
hostName = lib.mkOption {
type = lib.types.str;
description = "Public hostname used by clients to reach this Attic server.";
};
listenAddress = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
description = "Local address atticd binds to behind the reverse proxy.";
};
port = lib.mkOption {
type = lib.types.port;
default = 8080;
description = "Local port atticd binds to behind the reverse proxy.";
};
environmentFile = lib.mkOption {
type = lib.types.path;
description = ''
SOPS-backed environment file containing Attic JWT and object-storage
credentials.
'';
};
storage = {
bucket = lib.mkOption {
type = lib.types.str;
description = "Hetzner Object Storage bucket name used by Attic.";
};
endpoint = lib.mkOption {
type = lib.types.str;
description = "S3-compatible HTTPS endpoint for Hetzner Object Storage.";
};
region = lib.mkOption {
type = lib.types.str;
description = "Region name for Hetzner Object Storage.";
};
};
};
config = lib.mkIf cfg.enable {
services.atticd = {
enable = true;
environmentFile = cfg.environmentFile;
settings = {
listen = "${cfg.listenAddress}:${toString cfg.port}";
allowed-hosts = [ cfg.hostName ];
api-endpoint = "https://${cfg.hostName}/";
compression.type = "zstd";
storage = {
type = "s3";
bucket = cfg.storage.bucket;
endpoint = cfg.storage.endpoint;
region = cfg.storage.region;
};
};
};
};
}
+3 -1
View File
@@ -31,7 +31,7 @@ in {
locations."= /config.element.${matrixDomain}.json".return = "302 /config.json"; locations."= /config.element.${matrixDomain}.json".return = "302 /config.json";
root = pkgs.element-web.override { root = pkgs.hectic.element-web.override {
conf = { conf = {
default_server_config = { default_server_config = {
"m.homeserver".base_url = "https://${matrixDomain}"; "m.homeserver".base_url = "https://${matrixDomain}";
@@ -43,6 +43,8 @@ in {
matrixDomain matrixDomain
]; ];
hectic.videoMessages.enabled = true;
jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) { jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) {
preferred_domain = jitsiPreferredDomain; preferred_domain = jitsiPreferredDomain;
}; };
+224
View File
@@ -0,0 +1,224 @@
{ ... }: {
lib,
config,
...
}: let
cfg = config.hectic.services.ente;
webHostNames = [
cfg.domains.accounts
cfg.domains.cast
cfg.domains.photos
];
in {
options.hectic.services.ente = {
enable = lib.mkEnableOption "Ente Photos self-hosted service";
apiDomain = lib.mkOption {
type = lib.types.str;
description = "Public hostname for the Ente Museum API.";
};
domains = {
accounts = lib.mkOption {
type = lib.types.str;
description = "Public hostname for the Ente accounts web app.";
};
cast = lib.mkOption {
type = lib.types.str;
description = "Public hostname for the Ente cast web app.";
};
albums = lib.mkOption {
type = lib.types.str;
description = "Public hostname for public Ente album links.";
};
photos = lib.mkOption {
type = lib.types.str;
description = "Public hostname for the Ente Photos web app.";
};
};
maxUploadSize = lib.mkOption {
type = lib.types.str;
default = "10G";
description = "Maximum request body accepted by nginx in front of Museum.";
};
disableRegistration = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Whether Museum should reject new account registration.";
};
smtp = {
enable = lib.mkEnableOption "SMTP delivery for Ente verification emails";
host = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
description = "SMTP host Museum uses to send verification emails.";
};
port = lib.mkOption {
type = lib.types.port;
default = 25;
description = "SMTP port Museum uses to send verification emails.";
};
email = lib.mkOption {
type = lib.types.str;
description = "From email address used by Museum.";
};
senderName = lib.mkOption {
type = lib.types.str;
default = "Ente Photos";
description = "Display name used for Ente verification emails.";
};
encryption = lib.mkOption {
type = lib.types.nullOr (lib.types.enum [ "tls" "ssl" ]);
default = null;
description = "Optional SMTP encryption mode. Leave null for local plaintext SMTP.";
};
};
storage = {
bucket = lib.mkOption {
type = lib.types.str;
description = "S3-compatible bucket used by Ente for photo object storage.";
};
endpoint = lib.mkOption {
type = lib.types.str;
description = "S3-compatible endpoint URL.";
};
region = lib.mkOption {
type = lib.types.str;
description = "S3-compatible region name.";
};
hotStorage = lib.mkOption {
type = lib.types.enum [
"b2-eu-cen"
"wasabi-eu-central-2-v3"
"scw-eu-fr-v3"
];
default = "b2-eu-cen";
description = ''
Museum's primary hot-storage key. Upstream requires one of its
historical S3 storage identifiers even when the backing provider is a
generic S3-compatible service.
'';
};
usePathStyleUrls = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Whether Museum should use path-style S3 URLs.";
};
};
secrets = {
encryptionKeyFile = lib.mkOption {
type = lib.types.path;
description = "File containing Museum key.encryption.";
};
hashKeyFile = lib.mkOption {
type = lib.types.path;
description = "File containing Museum key.hash.";
};
jwtSecretFile = lib.mkOption {
type = lib.types.path;
description = "File containing Museum jwt.secret.";
};
s3AccessKeyFile = lib.mkOption {
type = lib.types.path;
description = "File containing the S3 access key.";
};
s3SecretKeyFile = lib.mkOption {
type = lib.types.path;
description = "File containing the S3 secret key.";
};
};
};
config = lib.mkIf cfg.enable {
services.ente = {
api = {
enable = true;
enableLocalDB = true;
domain = cfg.apiDomain;
nginx.enable = true;
settings = {
key = {
encryption._secret = cfg.secrets.encryptionKeyFile;
hash._secret = cfg.secrets.hashKeyFile;
};
jwt.secret._secret = cfg.secrets.jwtSecretFile;
s3 = {
hot_storage.primary = cfg.storage.hotStorage;
derived-storage = cfg.storage.hotStorage;
are_local_buckets = false;
use_path_style_urls = cfg.storage.usePathStyleUrls;
${cfg.storage.hotStorage} = {
key._secret = cfg.secrets.s3AccessKeyFile;
secret._secret = cfg.secrets.s3SecretKeyFile;
endpoint = cfg.storage.endpoint;
region = cfg.storage.region;
bucket = cfg.storage.bucket;
};
};
internal.disable-registration = cfg.disableRegistration;
smtp = lib.mkIf cfg.smtp.enable ({
inherit (cfg.smtp) host port email;
sender-name = cfg.smtp.senderName;
} // lib.optionalAttrs (cfg.smtp.encryption != null) {
encryption = cfg.smtp.encryption;
});
};
};
web = {
enable = true;
domains = {
api = cfg.apiDomain;
inherit (cfg.domains) accounts cast albums photos;
};
};
};
services.nginx.virtualHosts =
(lib.genAttrs webHostNames (_: {
enableACME = true;
forceSSL = true;
})) // {
${cfg.apiDomain} = {
enableACME = true;
forceSSL = true;
extraConfig = lib.mkForce ''
client_max_body_size ${cfg.maxUploadSize};
'';
locations."/".extraConfig = ''
proxy_read_timeout 600s;
proxy_send_timeout 600s;
'';
};
};
};
}
+12 -2
View File
@@ -56,8 +56,18 @@ in {
certificateScheme = "acme-nginx"; certificateScheme = "acme-nginx";
}; };
# NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records services.postfix.settings.main = {
services.postfix.settings.main.inet_protocols = lib.mkDefault "ipv4"; # NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records.
inet_protocols = lib.mkDefault "ipv4";
# NOTE(yukkop): nixos-mailserver enables DANE by default. Some large MXes
# currently fail certificate verification under this policy, which leaves
# otherwise valid transactional mail deferred in the queue. Keep STARTTLS
# opportunistic for outbound delivery rather than blocking mail entirely.
smtp_tls_security_level = lib.mkForce "may";
smtp_dns_support_level = lib.mkForce "enabled";
smtp_tls_policy_maps = lib.mkForce "";
};
security.acme.acceptTerms = true; security.acme.acceptTerms = true;
security.acme.defaults.email = "security@" + cfg.domain; security.acme.defaults.email = "security@" + cfg.domain;
+1 -1
View File
@@ -147,7 +147,7 @@ in {
(lib.mkIf cfg.watcher.enable { (lib.mkIf cfg.watcher.enable {
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault { sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
sopsFile = "${flake}/sus/sentinella-default.yaml"; sopsFile = flake + "/sus/sentinella-default.yaml";
}; };
systemd.services."sentinella-watcher" = { systemd.services."sentinella-watcher" = {
@@ -0,0 +1,153 @@
{ ... }: {
pkgs,
lib,
config,
...
}: let
cfg = config.hectic.services.stable-video-diffusion;
in {
options.hectic.services.stable-video-diffusion = {
enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
host = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
description = "Address the Stable Video Diffusion API binds to.";
};
port = lib.mkOption {
type = lib.types.port;
default = 7861;
description = "Port the Stable Video Diffusion API binds to.";
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.hectic.stable-video-diffusion-api;
defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
description = "Package providing the Stable Video Diffusion API executable.";
};
modelId = lib.mkOption {
type = lib.types.str;
default = "stabilityai/stable-video-diffusion-img2vid-xt";
description = "Model identifier loaded by the Stable Video Diffusion API.";
};
device = lib.mkOption {
type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
default = null;
description = ''
Torch device requested from the Stable Video Diffusion API. When null,
the package keeps its own auto-detection behavior.
'';
};
libraryPath = lib.mkOption {
type = with lib.types; listOf str;
default = [];
description = ''
Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
should include /run/opengl-driver/lib so libcuda.so is visible to torch.
'';
};
stateDir = lib.mkOption {
type = lib.types.str;
default = "/var/lib/stable-video-diffusion-api";
description = "Persistent state directory for the Stable Video Diffusion API.";
};
cacheDir = lib.mkOption {
type = lib.types.str;
default = "/var/cache/stable-video-diffusion-api";
description = "Cache directory for downloaded model and runtime artifacts.";
};
outputDir = lib.mkOption {
type = lib.types.str;
default = "${cfg.stateDir}/outputs";
defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
description = "Directory where generated video outputs are written.";
};
environmentFile = lib.mkOption {
type = with lib.types; nullOr path;
default = null;
description = ''
Optional environment file for secrets or runtime overrides. Values from
the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
and LD_LIBRARY_PATH by default.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Whether to open the API port in the firewall.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.device != "cuda" || cfg.libraryPath != [];
message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
}
];
users.users.stable-video-diffusion = {
isSystemUser = true;
group = "stable-video-diffusion";
};
users.groups.stable-video-diffusion = {};
systemd.tmpfiles.rules = [
"d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
"d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
"d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
];
systemd.services.stable-video-diffusion-api = {
description = "Stable Video Diffusion HTTP API";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
SVD_API_HOST = cfg.host;
SVD_API_PORT = toString cfg.port;
SVD_MODEL_ID = cfg.modelId;
SVD_STATE_DIR = cfg.stateDir;
SVD_CACHE_DIR = cfg.cacheDir;
SVD_OUTPUT_DIR = cfg.outputDir;
}
// lib.optionalAttrs (cfg.device != null) {
SVD_DEVICE = cfg.device;
}
// lib.optionalAttrs (cfg.libraryPath != []) {
LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
};
serviceConfig = lib.mkMerge [
{
Type = "simple";
User = "stable-video-diffusion";
Group = "stable-video-diffusion";
WorkingDirectory = cfg.stateDir;
ExecStart = lib.getExe' cfg.package "stable-video-diffusion-api";
Restart = "on-failure";
RestartSec = "5s";
TimeoutStopSec = "30s";
KillSignal = "SIGTERM";
KillMode = "mixed";
StandardOutput = "journal";
StandardError = "journal";
}
(lib.mkIf (cfg.environmentFile != null) {
EnvironmentFile = cfg.environmentFile;
})
];
};
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
};
}
@@ -1,230 +0,0 @@
# Matrix Cluster Failover Runbook (`accord.tube`)
Primary: `hectic-lab` (NL, `128.140.75.58`)
Standby: `bfs.poland.xray` (PL, `91.198.166.181`)
Module: `hectic.generic.matrix-cluster` (`nixos/module/generic/matrix-cluster.nix`).
Shared secrets: `sus/matrix-cluster.yaml`.
All `psql` and `pg_ctl` invocations use PostgreSQL **17** at data dir
`/var/lib/postgresql/17`.
## Initial setup
### 1. Provision shared SOPS file (`sus/matrix-cluster.yaml`)
On a workstation with both yukkop and yukkop-alt age keys available:
```sh
sudo cat /var/lib/matrix-synapse/homeserver.signing.key # on NL (hectic-lab)
# Copy the single line value into the buffer for the next step.
sops sus/matrix-cluster.yaml
```
Populate the editor with:
```yaml
matrix:
signing-key: <paste verbatim signing-key line from NL>
postgres-replication-password: <openssl rand -base64 32>
object-storage:
credentials: |
ACCESS_KEY_ID=<verbatim copy from sus/hectic-lab.yaml>
SECRET_ACCESS_KEY=<verbatim copy from sus/hectic-lab.yaml>
porkbun-api-key: <PORKBUN_API_KEY>
porkbun-secret-api-key: <PORKBUN_SECRET_API_KEY>
```
Verify recipients:
```sh
sops updatekeys sus/matrix-cluster.yaml
sops -d sus/matrix-cluster.yaml | grep -E 'signing-key|porkbun-api-key|object-storage'
```
Expected: all five keys present, exit 0.
### 2. Deploy NL primary first
```sh
nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux' --target-host root@128.140.75.58
```
Verify on NL:
```sh
sudo systemctl status matrix-synapse postgresql matrix-cluster-replication-password
sudo -u postgres psql -c "select rolname, rolreplication from pg_roles where rolname='replication';"
# Expected: replication | t
```
### 3. Seed PL replica with `pg_basebackup`
On PL:
```sh
sudo systemctl stop postgresql
sudo rm -rf /var/lib/postgresql/17
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
pg_basebackup \
-h 128.140.75.58 \
-p 5432 \
-U replication \
-D /var/lib/postgresql/17 \
-Fp -Xs -P -R \
--no-password
```
`-R` writes `standby.signal` and an initial `primary_conninfo`. The
matrix-cluster module's `matrix-cluster-standby-bootstrap` service will
overwrite `primary_conninfo` to use a libpq passfile on next boot.
### 4. Deploy PL standby
```sh
nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux' --target-host root@91.198.166.181
sudo systemctl start postgresql
```
Verify streaming on NL:
```sh
sudo -u postgres psql -c 'select client_addr, state, sync_state from pg_stat_replication;'
# Expected: 91.198.166.181 | streaming | async
```
Verify standby on PL:
```sh
sudo -u postgres psql -c 'select pg_is_in_recovery();'
# Expected: t
sudo systemctl is-active matrix-synapse
# Expected: inactive (standby keeps Synapse off)
```
### 5. Remove duplicate S3 credentials from `sus/hectic-lab.yaml`
Only AFTER NL is confirmed healthy reading from the new shared file:
```sh
sops sus/hectic-lab.yaml
# Delete the matrix/object-storage/credentials block.
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
```
## Normal operations
```sh
# NL: replication health
sudo -u postgres psql -c 'select * from pg_stat_replication;'
# Expected: 1 row, state=streaming, sync_state=async
# PL: replay status
sudo -u postgres psql -c 'select now() - pg_last_xact_replay_timestamp() as lag;'
# Both: cert renewal
sudo systemctl status acme-accord.tube.timer
sudo journalctl -u acme-accord.tube.service --since '24 hours ago'
# Synapse health (NL primary)
curl -sf https://accord.tube/_matrix/client/versions | head
```
## Planned failover (NL -> PL)
```sh
# 1. Drain NL: stop accepting writes.
sudo systemctl stop matrix-synapse
sudo systemctl stop postgresql # ensure no new WAL after this point
# 2. Promote PL replica.
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
# Wait until pg_is_in_recovery() returns f:
sudo -u postgres psql -c 'select pg_is_in_recovery();'
# 3. Make the role switch declarative before rebuilding.
# Edit the flake so rebuilds match the promoted database state:
# - nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
# hectic.generic.matrix-cluster.role = "primary";
# hectic.generic.matrix-cluster.overrideEnableSynapse = true;
# hectic.generic.matrix-cluster.secretsFile = config.sops.secrets."matrix/secrets".path;
# - nixos/system/hectic-lab/hectic-lab.nix:
# hectic.generic.matrix-cluster.role = "standby";
# hectic.generic.matrix-cluster.overrideEnableSynapse = false;
# hectic.generic.matrix-cluster.replication.peerHost = "91.198.166.181";
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "128.140.75.58/32" ];
# (You will also need a matrix/secrets entry on PL - copy from NL via SOPS.)
sudo nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux'
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
sudo systemctl status matrix-synapse
# 4. Swap DNS A record at Porkbun:
# accord.tube A 91.198.166.181 (was 128.140.75.58)
# TTL: set to 300 in advance of any planned failover.
# Porkbun UI: https://porkbun.com/account/domainsSpeedy -> accord.tube -> DNS -> edit A record.
# Or via API:
sudo curl -sX POST https://api.porkbun.com/api/json/v3/dns/editByNameType/accord.tube/A \
-H 'content-type: application/json' \
-d "$(jq -n --arg k "$PORKBUN_API_KEY" --arg s "$PORKBUN_SECRET_API_KEY" \
'{secretapikey:$s,apikey:$k,content:"91.198.166.181",ttl:"300"}')"
# 5. Federation smoke test.
curl -s 'https://federationtester.matrix.org/api/report?server_name=accord.tube' | jq .FederationOK
# Expected: true
```
Expected after the rebuilds:
- `bfs.poland.xray` evaluates and runs as `role = "primary"`.
- `hectic-lab` evaluates as `role = "standby"` with Synapse forced off.
- Future `nixos-rebuild` runs preserve the promoted topology instead of reapplying standby settings to PL.
## Failback (PL -> NL)
```sh
# 1. Stop NL postgres if still up; clear its data dir.
sudo systemctl stop postgresql matrix-synapse
sudo rm -rf /var/lib/postgresql/17
# 2. Re-seed NL from PL (now the live primary).
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
pg_basebackup -h 91.198.166.181 -p 5432 -U replication \
-D /var/lib/postgresql/17 -Fp -Xs -P -R --no-password
# 3. Temporarily flip roles in the flake:
# - hectic-lab.nix: role = "standby"; peerHost = "91.198.166.181";
# - bfs.poland.xray.nix: role = "primary"; peerHost = "128.140.75.58";
# Rebuild both.
# 4. Once NL is streaming green, do the reverse failover dance:
sudo systemctl stop matrix-synapse # on PL
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote # on NL
# Then revert the flake role assignments back to NL=primary / PL=standby and
# rebuild both hosts.
# 5. Swap DNS back at Porkbun (A -> 128.140.75.58).
```
## Disaster recovery (NL permanently lost)
```sh
# 1. Promote PL as the new permanent primary.
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
# 2. Edit nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
# hectic.generic.matrix-cluster.role = "primary";
# hectic.generic.matrix-cluster.overrideEnableSynapse = lib.mkForce null;
# hectic.generic.matrix-cluster.replication.peerHost = "<new-standby-ip>";
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "<new-standby-ip>/32" ];
# 3. Provision a new host (any region with Porkbun-managed DNS) and import
# self.nixosModules.matrix-cluster with role = "standby" pointed at PL's IP.
# 4. Bootstrap the new standby via pg_basebackup from PL exactly as in
# "Initial setup" step 3, replacing 128.140.75.58 with PL's IP.
# 5. Update Porkbun A record to PL's IP permanently.
```
@@ -9,16 +9,11 @@
config, config,
... ...
}: let }: let
matrixBackend = "https://128.140.75.58";
matrixHost = "accord.tube"; matrixHost = "accord.tube";
jitsiHost = "meet.accord.tube"; jitsiHost = "meet.accord.tube";
elementEntryDomain = "element.bfs.band"; elementEntryDomain = "element.bfs.band";
polandEntryDomain = "bfs.band"; polandEntryDomain = "bfs.band";
backendProxyConfig = '' matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
proxy_ssl_server_name on;
proxy_ssl_name ${matrixHost};
proxy_set_header Host ${matrixHost};
'';
in { in {
imports = [ imports = [
self.nixosModules.xray-system self.nixosModules.xray-system
@@ -46,7 +41,6 @@ in {
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path; credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
}; };
replication = { replication = {
peerHost = "128.140.75.58";
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path; passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
}; };
acme = { acme = {
@@ -71,11 +65,31 @@ in {
hostName = jitsiHost; hostName = jitsiHost;
}; };
# NOTE(yukkop): disk was provisioned outside disko, so the expected partition
# label does not exist. Pin root to the live filesystem UUID so stage 1 can
# mount `/` reliably.
fileSystems."/" = lib.mkForce {
device = "/dev/disk/by-uuid/06b48ef1-a1eb-428d-821c-90c96a624542";
fsType = "ext4";
};
security.acme = { security.acme = {
acceptTerms = true; acceptTerms = true;
defaults.email = "security@bfs.band"; defaults.email = "security@bfs.band";
}; };
# NOTE(yukkop): this host gets an IPv6 route via RA, but object storage
# fetches to hel1.your-objectstorage.com currently stall over IPv6 while
# IPv4 works. Synapse's S3 media backend uses getaddrinfo ordering, so
# prefer IPv4 here to keep Element media downloads responsive.
environment.etc."gai.conf".text = ''
precedence ::ffff:0:0/96 100
'';
systemd.services.matrix-synapse.restartTriggers = [
config.environment.etc."gai.conf".source
];
services.nginx = { services.nginx = {
enable = true; enable = true;
@@ -112,19 +126,17 @@ in {
}; };
locations."= /livekit/jwt" = { locations."= /livekit/jwt" = {
proxyPass = "${matrixBackend}/livekit/jwt"; proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
extraConfig = backendProxyConfig;
}; };
locations."^~ /livekit/jwt/" = { locations."^~ /livekit/jwt/" = {
proxyPass = "${matrixBackend}/livekit/jwt/"; proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
extraConfig = backendProxyConfig;
}; };
locations."= /livekit/sfu" = { locations."= /livekit/sfu" = {
proxyPass = "${matrixBackend}/livekit/sfu"; proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
proxyWebsockets = true; proxyWebsockets = true;
extraConfig = backendProxyConfig + '' extraConfig = ''
proxy_send_timeout 120; proxy_send_timeout 120;
proxy_read_timeout 120; proxy_read_timeout 120;
proxy_buffering off; proxy_buffering off;
@@ -135,9 +147,9 @@ in {
}; };
locations."^~ /livekit/sfu/" = { locations."^~ /livekit/sfu/" = {
proxyPass = "${matrixBackend}/livekit/sfu/"; proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
proxyWebsockets = true; proxyWebsockets = true;
extraConfig = backendProxyConfig + '' extraConfig = ''
proxy_send_timeout 120; proxy_send_timeout 120;
proxy_read_timeout 120; proxy_read_timeout 120;
proxy_buffering off; proxy_buffering off;
@@ -148,13 +160,14 @@ in {
}; };
locations."^~ /_matrix/" = { locations."^~ /_matrix/" = {
proxyPass = "${matrixBackend}/_matrix/"; proxyPass = "http://127.0.0.1:8008";
extraConfig = backendProxyConfig; extraConfig = ''
client_max_body_size ${config.hectic.generic.matrix-cluster.maxUploadSize};
'';
}; };
locations."^~ /_synapse/client/" = { locations."^~ /_synapse/client/" = {
proxyPass = "${matrixBackend}/_synapse/client/"; proxyPass = "http://127.0.0.1:8008";
extraConfig = backendProxyConfig;
}; };
}; };
@@ -164,7 +177,7 @@ in {
locations."= /config.${elementEntryDomain}.json".return = "302 /config.json"; locations."= /config.${elementEntryDomain}.json".return = "302 /config.json";
root = pkgs.element-web.override { root = pkgs.hectic.element-web.override {
conf = { conf = {
default_server_config = { default_server_config = {
"m.homeserver".base_url = "https://${polandEntryDomain}"; "m.homeserver".base_url = "https://${polandEntryDomain}";
@@ -176,6 +189,8 @@ in {
preferred_domain = jitsiHost; preferred_domain = jitsiHost;
}; };
hectic.videoMessages.enabled = true;
room_directory.servers = [ matrixHost ]; room_directory.servers = [ matrixHost ];
default_theme = "dark"; default_theme = "dark";
@@ -189,41 +204,41 @@ in {
key = "matrix/signing-key"; key = "matrix/signing-key";
owner = "matrix-synapse"; owner = "matrix-synapse";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
sops.secrets."matrix/postgres-replication-password" = { sops.secrets."matrix/postgres-replication-password" = {
key = "matrix/postgres-replication-password"; key = "matrix/postgres-replication-password";
owner = "postgres"; owner = "postgres";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
sops.secrets."matrix/object-storage/credentials" = { sops.secrets."matrix/object-storage/credentials" = {
key = "matrix/object-storage/credentials"; key = "matrix/object-storage/credentials";
owner = "matrix-synapse"; owner = "matrix-synapse";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
sops.secrets."matrix/secrets" = { sops.secrets."matrix/secrets" = {
key = "matrix/secrets"; key = "matrix/secrets";
owner = "matrix-synapse"; owner = "matrix-synapse";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
sops.secrets."matrix/turn-secret" = { sops.secrets."matrix/turn-secret" = {
key = "matrix/turn-secret"; key = "matrix/turn-secret";
owner = "turnserver"; owner = "turnserver";
group = "turnserver"; group = "turnserver";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
sops.secrets."matrix/porkbun-api-key" = { sops.secrets."matrix/porkbun-api-key" = {
key = "matrix/porkbun-api-key"; key = "matrix/porkbun-api-key";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
sops.secrets."matrix/porkbun-secret-api-key" = { sops.secrets."matrix/porkbun-secret-api-key" = {
key = "matrix/porkbun-secret-api-key"; key = "matrix/porkbun-secret-api-key";
mode = "0400"; mode = "0400";
sopsFile = "${flake}/sus/matrix-cluster.yaml"; sopsFile = matrixClusterSopsFile;
}; };
} }
+30
View File
@@ -0,0 +1,30 @@
{
domain,
...
}: {
config,
...
}: {
hectic.services.attic = {
enable = true;
hostName = "cache.${domain}";
port = 8081;
environmentFile = config.sops.secrets."atticd/environment".path;
storage = {
bucket = "cache-hectic-lab";
endpoint = "https://hel1.your-objectstorage.com";
region = "hel1";
};
};
services.nginx.virtualHosts."cache.${domain}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
client_max_body_size 0;
'';
locations."/" = {
proxyPass = "http://127.0.0.1:8081";
};
};
}
+42
View File
@@ -0,0 +1,42 @@
{
domain,
...
}: {
config,
...
}: let
enteDomain = "ente.${domain}";
in {
hectic.services.ente = {
enable = true;
apiDomain = "api.${enteDomain}";
disableRegistration = false;
domains = {
accounts = "accounts.${enteDomain}";
cast = "cast.${enteDomain}";
albums = "albums.${enteDomain}";
photos = "photos.${enteDomain}";
};
smtp = {
enable = true;
host = "mail.${domain}";
email = "security@${domain}";
};
storage = {
bucket = "ente-hectic-lab";
endpoint = "https://hel1.your-objectstorage.com";
region = "hel1";
};
secrets = {
encryptionKeyFile = config.sops.secrets."ente/key-encryption".path;
hashKeyFile = config.sops.secrets."ente/key-hash".path;
jwtSecretFile = config.sops.secrets."ente/jwt-secret".path;
s3AccessKeyFile = config.sops.secrets."ente/s3-access-key".path;
s3SecretKeyFile = config.sops.secrets."ente/s3-secret-key".path;
};
};
}
@@ -0,0 +1,74 @@
{ pkgs, ... }:
let
port = 22222;
stateDir = "/var/lib/experimental-sshd";
configFile = pkgs.writeText "experimental-sshd_config" ''
Port 22222
ListenAddress 0.0.0.0
ListenAddress ::
HostKey ${stateDir}/ssh_host_ed25519_key
HostKey ${stateDir}/ssh_host_rsa_key
PasswordAuthentication no
KbdInteractiveAuthentication no
PubkeyAuthentication yes
PermitRootLogin prohibit-password
UsePAM yes
AuthenticationMethods publickey
AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u
LogLevel DEBUG3
VersionAddendum none
HostKeyAlgorithms rsa-sha2-512,rsa-sha2-256,ssh-ed25519
KexAlgorithms curve25519-sha256,diffie-hellman-group14-sha256
Ciphers aes256-ctr,aes128-ctr
MACs hmac-sha2-512,hmac-sha2-256
'';
keygenScript = pkgs.writeShellScript "experimental-sshd-keygen" ''
set -eu
${pkgs.coreutils}/bin/mkdir -p -m 0700 ${stateDir}
if [ ! -f ${stateDir}/ssh_host_ed25519_key ]; then
${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f ${stateDir}/ssh_host_ed25519_key -N ""
fi
if [ ! -f ${stateDir}/ssh_host_rsa_key ]; then
${pkgs.openssh}/bin/ssh-keygen -t rsa -b 4096 -f ${stateDir}/ssh_host_rsa_key -N ""
fi
'';
in
{
environment.etc."ssh/experimental-sshd_config".source = configFile;
networking.firewall.allowedTCPPorts = [ port ];
systemd.services.experimental-sshd-keygen = {
description = "Generate experimental SSH host keys";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
StateDirectory = "experimental-sshd";
ExecStart = keygenScript;
};
};
systemd.services.experimental-sshd = {
description = "Experimental SSH daemon";
wantedBy = [ "multi-user.target" ];
wants = [ "experimental-sshd-keygen.service" ];
after = [ "network.target" "experimental-sshd-keygen.service" ];
serviceConfig = {
Type = "simple";
StateDirectory = "experimental-sshd";
RuntimeDirectory = "experimental-sshd";
ExecStart = "${pkgs.openssh}/bin/sshd -D -e -f /etc/ssh/experimental-sshd_config";
};
preStart = ''
${pkgs.openssh}/bin/sshd -t -f /etc/ssh/experimental-sshd_config
'';
};
}
+39 -28
View File
@@ -15,7 +15,7 @@ with builtins;
with lib; with lib;
let let
domain = "hectic-lab.com"; domain = "hectic-lab.com";
matrixDomain = "accord.tube"; sshPort = 22;
mailUserNames = [ mailUserNames = [
"security" "security"
"founders" "founders"
@@ -25,6 +25,7 @@ let
"iana-perlyk" "iana-perlyk"
"snuff" "snuff"
"antoshka" "antoshka"
"evgenii-kazakov"
]; ];
mkMailPasswordSecret = name: { mkMailPasswordSecret = name: {
name = "mailserver/${name}/hashedPassword"; name = "mailserver/${name}/hashedPassword";
@@ -36,9 +37,12 @@ let
hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path; hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path;
}; };
}; };
sslOpts = { mkEnteSecret = name: {
sslCertificate = config.sops.secrets."ssl/porkbun/${domain}/domain.cert.pem".path; name = "ente/${name}";
sslCertificateKey = config.sops.secrets."ssl/porkbun/${domain}/private.key.pem".path; value = {
owner = "ente";
group = "ente";
};
}; };
in { in {
imports = [ imports = [
@@ -51,9 +55,12 @@ in {
inputs.hectic-landing.nixosModules.hectic-landing inputs.hectic-landing.nixosModules.hectic-landing
(import ./attic.nix { inherit flake self inputs domain; })
(import ./containers.nix { inherit flake self inputs; }) (import ./containers.nix { inherit flake self inputs; })
(import ./mechabellum.nix { inherit flake self inputs domain sslOpts; }) ./experimental-sshd.nix
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain sslOpts; }) (import ./ente.nix { inherit domain; })
(import ./mechabellum.nix { inherit flake self inputs domain; })
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
]; ];
services.hectic-landing = { services.hectic-landing = {
@@ -74,6 +81,7 @@ in {
enable = true; enable = true;
networkMatchConfigName = "enp1s0"; networkMatchConfigName = "enp1s0";
ipv4 = "128.140.75.58"; ipv4 = "128.140.75.58";
floatingIpv4 = "78.47.243.0";
ipv6 = "2a01:4f8:c2c:d54a"; ipv6 = "2a01:4f8:c2c:d54a";
}; };
services.matrix = { services.matrix = {
@@ -100,6 +108,7 @@ in {
''; '';
environment.systemPackages = with pkgs; [ environment.systemPackages = with pkgs; [
tcpdump
git git
rsync rsync
python311 python311
@@ -110,28 +119,25 @@ in {
sops = { sops = {
gnupg.sshKeyPaths = [ ]; gnupg.sshKeyPaths = [ ];
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
defaultSopsFile = "${flake}/sus/hectic-lab.yaml"; defaultSopsFile = flake + "/sus/hectic-lab.yaml";
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // { secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
"init-postgresql" = { "init-postgresql" = {
key = "init-postgresql"; key = "init-postgresql";
}; };
"ssl/porkbun/${domain}/domain.cert.pem" = { "atticd/environment" = {};
group = "nginx";
mode = "0440";
};
"ssl/porkbun/${domain}/private.key.pem" = {
group = "nginx";
mode = "0440";
};
"ssl/porkbun/${domain}/public.key.pem" = {
group = "nginx";
mode = "0440";
};
"wg-bfs/private-key" = {}; "wg-bfs/private-key" = {};
}; } // builtins.listToAttrs (map mkEnteSecret [
"key-encryption"
"key-hash"
"jwt-secret"
"s3-access-key"
"s3-secret-key"
]);
}; };
users.users.root.openssh.authorizedKeys.keys = [ users.users.root.openssh.authorizedKeys.keys = [
# neuro machine
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
# yukkop # yukkop
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
@@ -158,6 +164,8 @@ in {
]; ];
}; };
services.openssh.ports = [ sshPort ];
services.mailserver = { services.mailserver = {
enable = true; enable = true;
domain = domain; domain = domain;
@@ -178,10 +186,10 @@ in {
networking.firewall = { networking.firewall = {
allowedTCPPorts = [ allowedTCPPorts = [
sshPort # ssh
80 80
443 443
3306 # mysql 3306 # mysql
11012 # gitea ssh
25565 25565
55228 # ss-bfs 55228 # ss-bfs
]; ];
@@ -204,8 +212,8 @@ in {
services.nginx = { services.nginx = {
enable = true; enable = true;
# NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module # NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module
# (ACME-managed). See services.hectic-landing above. virtualHosts."store.${domain}" = {
virtualHosts."store.${domain}" = sslOpts // { enableACME = true;
forceSSL = true; forceSSL = true;
root = "/var/www/store"; root = "/var/www/store";
locations."/" = { locations."/" = {
@@ -214,7 +222,8 @@ in {
''; '';
}; };
}; };
virtualHosts."snuff.${domain}" = sslOpts // { virtualHosts."snuff.${domain}" = {
enableACME = true;
forceSSL = true; forceSSL = true;
locations."/" = { locations."/" = {
extraConfig = '' extraConfig = ''
@@ -223,7 +232,8 @@ in {
''; '';
}; };
}; };
virtualHosts."nrv.${domain}" = sslOpts // { virtualHosts."nrv.${domain}" = {
enableACME = true;
forceSSL = true; forceSSL = true;
locations."/" = { locations."/" = {
extraConfig = '' extraConfig = ''
@@ -232,7 +242,8 @@ in {
''; '';
}; };
}; };
virtualHosts."yukkop.${domain}" = sslOpts // { virtualHosts."yukkop.${domain}" = {
enableACME = true;
forceSSL = true; forceSSL = true;
locations."/" = { locations."/" = {
extraConfig = '' extraConfig = ''
@@ -257,10 +268,10 @@ in {
gitea = { gitea = {
enable = true; enable = true;
package = pkgs.hectic.gitea-heatmap; package = pkgs.hectic.gitea-heatmap;
settings.service.DISABLE_REGISTRATION = false; settings.service.DISABLE_REGISTRATION = true;
settings.server = { settings.server = {
HTTP_PORT = 11011; HTTP_PORT = 11011;
#SSH_PORT = 22; SSH_PORT = sshPort;
SSH_DOMAIN = "hectic-lab.com"; SSH_DOMAIN = "hectic-lab.com";
}; };
database = { database = {
+3 -2
View File
@@ -1,7 +1,6 @@
{ {
inputs, inputs,
domain, domain,
sslOpts,
... ...
}: { }: {
pkgs, pkgs,
@@ -29,7 +28,8 @@ in {
enable = true; enable = true;
}; };
services.nginx.virtualHosts."${mechDomain}" = sslOpts // { services.nginx.virtualHosts."${mechDomain}" = {
enableACME = true;
forceSSL = true; forceSSL = true;
root = inputs.mechabellum-replay-analysis.packages.${system}.frontend; root = inputs.mechabellum-replay-analysis.packages.${system}.frontend;
@@ -37,6 +37,7 @@ in {
proxyPass = "http://${apiHost}:${builtins.toString apiPort}"; proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
extraConfig = '' extraConfig = ''
proxy_http_version 1.1; proxy_http_version 1.1;
client_max_body_size 500M;
''; '';
}; };
+2 -2
View File
@@ -3,7 +3,6 @@
flake, flake,
self, self,
domain, domain,
sslOpts,
... ...
}: { ... }: { }: { ... }: {
hectic.services."sentinèlla" = { hectic.services."sentinèlla" = {
@@ -17,7 +16,8 @@
}; };
services.nginx = { services.nginx = {
virtualHosts."probe.${domain}" = sslOpts // { virtualHosts."probe.${domain}" = {
enableACME = true;
forceSSL = true; forceSSL = true;
locations."/" = { locations."/" = {
proxyPass = "http://127.0.0.1:5988"; proxyPass = "http://127.0.0.1:5988";
+2 -22
View File
@@ -15,32 +15,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
self.overlays.default self.overlays.default
inputs.nix-minecraft.overlay inputs.nix-minecraft.overlay
]; ];
config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [ config.allowUnfreePredicate = pkg:
self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
"minecraft-server" "minecraft-server"
"neoforge" "neoforge"
"nvidia-x11" "nvidia-x11"
"cuda_nvcc"
"cuda_cudart"
"cuda_cuobjdump"
"cuda_cupti"
"cuda_nvdisasm"
"cuda_cccl"
"cuda_nvml_dev"
"cuda_nvrtc"
"cuda_nvtx"
"cuda_profiler_api"
"libcusparse_lt"
"libcublas"
"libcufft"
"libcufile"
"libcurand"
"libcusolver"
"libnvjitlink"
"libcusparse"
"cudnn"
]; ];
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x) # jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
config.permittedInsecurePackages = [ config.permittedInsecurePackages = [
+19
View File
@@ -17,6 +17,11 @@
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13"; ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
pkgs.stdenv.cc.cc.lib
pkgs.zlib
];
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation { ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
pname = "ollama"; pname = "ollama";
version = "0.22.1"; version = "0.22.1";
@@ -176,6 +181,19 @@ in {
openFirewall = false; openFirewall = false;
}; };
hectic.services.stable-video-diffusion = {
enable = true;
host = "127.0.0.1";
port = 7861;
package = pkgs.hectic.stable-video-diffusion-api;
device = "cuda";
libraryPath = [
stableVideoDiffusionLibraryPath
"/run/opengl-driver/lib"
];
openFirewall = false;
};
networking = { networking = {
networkmanager.enable = true; networkmanager.enable = true;
useDHCP = lib.mkDefault true; useDHCP = lib.mkDefault true;
@@ -263,6 +281,7 @@ in {
in [ in [
#python-ai #python-ai
git git
hectic.hiddify-core
neovim neovim
wget wget
ethtool ethtool
+20
View File
@@ -0,0 +1,20 @@
{
flake,
self,
inputs,
system ? "x86_64-linux",
...
}: let
# Use folder name as name of this system
name = builtins.baseNameOf ./.;
in self.lib.nixpkgs-lib.nixosSystem {
pkgs = import inputs.nixpkgs {
inherit system;
overlays = [ self.overlays.default ];
};
modules = [
{ networking.hostName = name; }
(import ./${name}.nix { inherit flake self inputs; })
];
}
+35
View File
@@ -0,0 +1,35 @@
{
inputs,
self,
...
}: {
pkgs,
modulesPath,
...
}: {
imports = [
(modulesPath + "/profiles/qemu-guest.nix")
inputs.sops-nix.nixosModules.sops
self.nixosModules.hectic
];
hectic = {
archetype.dev.enable = true;
hardware.hetzner-cloud = {
enable = true;
device = "/dev/sda";
networkMatchConfigMac = "92:00:08:4a:b0:32";
ipv4 = "46.225.237.218";
ipv6 = "2a01:4f8:c2c:3b14";
};
};
users.users.root.openssh.authorizedKeys.keys = [
''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
];
environment.systemPackages = with pkgs; [
git
rsync
];
}
+37 -11
View File
@@ -1,18 +1,21 @@
# db-tool # db-tool
PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries. PostgreSQL utility package exposing separate development and operations binaries.
`db-dev` preserves the current development workflow via the `database` binary;
`db-ops` provides target-safe operational helpers such as secrets hydration.
## Provided Binaries ## Provided Binaries
| Binary | Description | | Binary | Description |
| --- | --- | | --- | --- |
| `database` | Main script for managing migrations, deployments, and logs. | | `database` | Main `db-dev` script for managing local development databases, migrations, deployments, and logs. |
| `db-ops` | Operational helper binary for target/runtime database actions. |
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. | | `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. | | `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
## Required Environment Variables ## Required Environment Variables
These variables must be set for `db-tool` to function. These variables must be set for `db-dev` / `database` to function.
| Variable | Description | | Variable | Description |
| --- | --- | | --- | --- |
@@ -39,7 +42,7 @@ These variables must be set for `db-tool` to function.
## Postgres Package Override ## Postgres Package Override
By default, `db-tool`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output: By default, `db-dev`/`db-ops`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
```nix ```nix
let let
@@ -48,7 +51,8 @@ let
]); ]);
in { in {
packages = [ packages = [
(pkgs.hectic."db-tool".override { postgresql = myPg; }) (pkgs.hectic."db-dev".override { postgresql = myPg; })
(pkgs.hectic."db-ops".override { postgresql = myPg; })
(pkgs.hectic."postgres-init".override { postgresql = myPg; }) (pkgs.hectic."postgres-init".override { postgresql = myPg; })
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; }) (pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
]; ];
@@ -66,6 +70,28 @@ The `pull_staging` subcommand allows importing data from a remote staging enviro
If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr. If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr.
## normalize-backup Contract
The `normalize-backup` subcommand converts a physical PostgreSQL backup into a
local-development restore artifact without modifying the input backup:
```sh
database normalize-backup [OPTIONS] [path]
```
The input `path` defaults to `${LOCAL_DIR}/focus/postgresql-backup` and must be a
backup directory compatible with `database restore`, containing `base.tar.gz`
and optionally `pg_wal.tar.gz`. The output defaults to a normalized backup path
and can be overridden with `--output <path>`. The output directory must not be
the same path as the input and must not be inside the input directory.
Normalization extracts the physical backup into temporary PGDATA, replaces
production PostgreSQL access/configuration with local restore-safe
`postgresql.conf` and `pg_hba.conf` files, removes standby/recovery/runtime
leftovers, starts the cluster locally, ensures the requested `--role` and
`--database` exist, stops cleanly, and repacks a backup directory that can be
used by `database restore`.
## Subcommands ## Subcommands
- `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success. - `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success.
@@ -73,20 +99,21 @@ If any of these variables are missing when `pull_staging` is invoked, the tool w
- `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`. - `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`.
- `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster. - `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster.
- `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`. - `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`.
- `normalize-backup`: Rewrite a physical backup artifact for local restore use.
- `pull_staging`: Import data from the staging environment based on the env contract. - `pull_staging`: Import data from the staging environment based on the env contract.
- `init`: Wrapper around `postgres-init` to start the cluster. - `init`: Wrapper around `postgres-init` to start the cluster.
- `migrator`: Directly invoke the migration tool with the correct environment context. - `migrator`: Directly invoke the migration tool with the correct environment context.
## shellHook Example ## shellHook Example
To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`: To use `db-dev` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
```nix ```nix
{ {
# ... # ...
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
packages = [ packages = [
pkgs.hectic.db-tool pkgs.hectic.db-dev
pkgs.hectic.postgres-init pkgs.hectic.postgres-init
pkgs.hectic.postgres-cleanup pkgs.hectic.postgres-cleanup
]; ];
@@ -111,7 +138,7 @@ To use `db-tool` in a Nix development shell, add the following to your `flake.ni
## hectic Bundle ## hectic Bundle
`db-tool` and `migrator` apply a single bundle of SQL files that bootstrap the `db-dev`, `db-ops`, and `migrator` apply a single bundle of SQL files that bootstrap the
`hectic` schema. The bundle lives in `hectic` schema. The bundle lives in
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full [`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
contract, file layout, and the `self.lib.hectic.*` Nix API. contract, file layout, and the `self.lib.hectic.*` Nix API.
@@ -167,6 +194,7 @@ ALTER DATABASE mydb SET hectic.inheritance_extra_excluded_schemas = 'legacy,etl'
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. | | `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. | | `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. | | `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. |
The bundle is idempotent — repeated application is safe. The bundle is idempotent — repeated application is safe.
@@ -187,15 +215,13 @@ directly against the paths exposed by `self.lib.hectic.*.path`:
```nix ```nix
services.postgresql.initialScript = pkgs.writeText "hectic-init.sql" '' services.postgresql.initialScript = pkgs.writeText "hectic-init.sql" ''
\i ${self.lib.hectic.version.path}
\i ${self.lib.hectic.secret.path} \i ${self.lib.hectic.secret.path}
\i ${self.lib.hectic.migration.path} \i ${self.lib.hectic.migration.path}
\i ${self.lib.hectic.inheritance.path} \i ${self.lib.hectic.inheritance.path}
''; '';
``` ```
The version file (`self.lib.hectic.version`) is templated and only exposes
`.sql` (a string). Materialize it with `pkgs.writeText` if a path is needed.
## Exit Codes ## Exit Codes
| Code | Meaning | | Code | Meaning |
@@ -4,6 +4,7 @@
: "${REMAINING_ARGS:=}" : "${REMAINING_ARGS:=}"
: "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}" : "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}"
: "${DEFAULT_NORMALIZED_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup-normalized}"
: "${SCRIPT_NAME:=$(basename "$0")}" : "${SCRIPT_NAME:=$(basename "$0")}"
SCRIPT_NAME=${SCRIPT_NAME%%.sh} SCRIPT_NAME=${SCRIPT_NAME%%.sh}
@@ -186,6 +187,9 @@ ${BGREEN}Database Subcommands:${NC}
${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup ${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup
${BCYAN}normalize-backup${NC} ${CYAN}[OPTIONS] [PATH]$NC
Normalize a raw physical backup for local restore/diff
${BCYAN}backup${NC} Create database backup ${BCYAN}backup${NC} Create database backup
Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC
@@ -246,6 +250,7 @@ ${BGREEN}Examples:${NC}
$SCRIPT_NAME init Initialize PostgreSQL only $SCRIPT_NAME init Initialize PostgreSQL only
$SCRIPT_NAME restore Restore from default backup $SCRIPT_NAME restore Restore from default backup
$SCRIPT_NAME restore /path/to/backup Restore from specific path $SCRIPT_NAME restore /path/to/backup Restore from specific path
$SCRIPT_NAME normalize-backup /path/to/raw-backup
$SCRIPT_NAME backup Create backup $SCRIPT_NAME backup Create backup
$SCRIPT_NAME log Show database logs $SCRIPT_NAME log Show database logs
$SCRIPT_NAME log list List available log files $SCRIPT_NAME log list List available log files
@@ -421,6 +426,43 @@ EOF
)" | "$PAGER_OR_CAT" )" | "$PAGER_OR_CAT"
} }
help_normalize_backup() {
# shellcheck disable=SC2059
printf "$(cat <<EOF
${BGREEN}Usage:${NC} $SCRIPT_NAME normalize-backup [OPTIONS] [path]
Normalize a raw PostgreSQL physical backup for local development.
This command leaves the input backup untouched. It extracts $BBLACK\`base.tar.gz\`$NC
and optional $BBLACK\`pg_wal.tar.gz\`$NC into temporary PGDATA, writes local
$BBLACK\`postgresql.conf\`$NC and $BBLACK\`pg_hba.conf\`$NC files, removes standby,
recovery, and runtime leftovers, starts the cluster locally, ensures a login role
and database exist, stops cleanly, then repacks a normalized backup directory.
${BGREEN}Arguments:${NC}
${BCYAN}path$NC Input backup directory (optional)
Defaults to $BBLACK$DEFAULT_BACKUP_PATH$NC
${BGREEN}Options:${NC}
$BCYAN-o$NC, $BCYAN--output$NC ${CYAN}<path>$NC Output backup directory
Defaults to $BBLACK$DEFAULT_NORMALIZED_BACKUP_PATH$NC
$BCYAN--role$NC ${CYAN}<name>$NC Login role to ensure (default $BBLACK\$(id -un)$NC)
$BCYAN--database$NC ${CYAN}<name>$NC Database to ensure (default $BBLACK\${PG_DATABASE:-testdb}$NC)
$BCYAN--admin-role$NC ${CYAN}<name>$NC Role used for local maintenance
Defaults to $BBLACK\${URI_USER:-postgres}$NC
$BCYAN-h$NC, $BCYAN--help$NC Show this help message
${BGREEN}Files Created:${NC}
- ${BBLACK}\`base.tar.gz\`$NC: Normalized database cluster backup
${BGREEN}Examples:${NC}
$SCRIPT_NAME normalize-backup /path/to/raw-backup
$SCRIPT_NAME normalize-backup --output focus/postgresql-backup-normalized
EOF
)" | "$PAGER_OR_CAT"
}
help_diff() { help_diff() {
# shellcheck disable=SC2059 # shellcheck disable=SC2059
printf "$(cat <<EOF printf "$(cat <<EOF
@@ -441,6 +483,8 @@ ${BGREEN}Arguments:
${BGREEN}Options:${NC} ${BGREEN}Options:${NC}
$BCYAN--tables${NC} ${CYAN}<list>${NC} Comma-separated list of tables to diff $BCYAN--tables${NC} ${CYAN}<list>${NC} Comma-separated list of tables to diff
Example: --tables users,orders,products Example: --tables users,orders,products
$BCYAN--ignore-migration-fail${NC}
Continue diff even if DB1 migrations fail
$BCYAN-m${NC}, $BCYAN--mock${NC} Mock external API calls when hydrating DB2 $BCYAN-m${NC}, $BCYAN--mock${NC} Mock external API calls when hydrating DB2
Replaces HTTP-calling functions with stubs/test data Replaces HTTP-calling functions with stubs/test data
$BCYAN-h${NC}, $BCYAN--help${NC} Show this help message $BCYAN-h${NC}, $BCYAN--help${NC} Show this help message
@@ -464,6 +508,7 @@ ${BGREEN}Examples:${NC}
$SCRIPT_NAME diff Compare using default backup $SCRIPT_NAME diff Compare using default backup
$SCRIPT_NAME diff /path/to/backup Compare using specific backup $SCRIPT_NAME diff /path/to/backup Compare using specific backup
$SCRIPT_NAME diff --tables users,orders Compare specific tables $SCRIPT_NAME diff --tables users,orders Compare specific tables
$SCRIPT_NAME diff --ignore-migration-fail Continue despite DB1 migration failure
$SCRIPT_NAME diff -m Compare with external APIs mocked $SCRIPT_NAME diff -m Compare with external APIs mocked
$SCRIPT_NAME diff log Show logs from diff operation $SCRIPT_NAME diff log Show logs from diff operation
@@ -805,6 +850,11 @@ subcommand_restore() {
RESTORE_BACKUP_PATH="$DEFAULT_BACKUP_PATH" RESTORE_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
fi fi
local restore_database="${PG_DATABASE:-}"
if [ -f "${RESTORE_BACKUP_PATH:?}/database_name" ]; then
restore_database=$(tr -d '\n' < "${RESTORE_BACKUP_PATH:?}/database_name")
fi
postgres-cleanup postgres-cleanup
local data="${PG_WORKING_DIR:?}/data" local data="${PG_WORKING_DIR:?}/data"
@@ -817,12 +867,243 @@ subcommand_restore() {
tar -xzf "${RESTORE_BACKUP_PATH:?}/pg_wal.tar.gz" -C "${data}/pg_wal" tar -xzf "${RESTORE_BACKUP_PATH:?}/pg_wal.tar.gz" -C "${data}/pg_wal"
fi fi
env PG_REUSE= postgres-init env PG_REUSE= PG_DATABASE="$restore_database" postgres-init
rm -f "${data}/standby.signal" "${data}/recovery.signal" rm -f "${data}/standby.signal" "${data}/recovery.signal"
restore_namespace restore_namespace
} }
___sql_literal() {
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/''/g")"
}
___normalize_backup_remove_leftovers() {
local pgdata="$1"
rm -f \
"$pgdata/postmaster.pid" \
"$pgdata/postmaster.opts" \
"$pgdata/recovery.signal" \
"$pgdata/standby.signal" \
"$pgdata/backup_label.old"
rm -f "$pgdata/pg_wal/archive_status"/*.ready 2>/dev/null || :
}
___normalize_backup_cleanup() {
local tmpdir="$1"
local pgdata="$2"
if [ -n "$pgdata" ] && [ -d "$pgdata" ]; then
pg_ctl -D "$pgdata" -m fast -w stop >/dev/null 2>&1 || :
fi
if [ -n "$tmpdir" ]; then
rm -rf "$tmpdir"
fi
}
___normalize_backup_refresh_collation() {
local sockdir="$1"
local port="$2"
local admin_role="$3"
local database_name="$4"
psql -h "$sockdir" -p "$port" -U "$admin_role" -d postgres \
-v ON_ERROR_STOP=1 -c "ALTER DATABASE \"$database_name\" REFRESH COLLATION VERSION;" \
>/dev/null 2>&1 || :
}
subcommand_normalize_backup() {
change_namespace 'db normalize-backup'
NORMALIZE_INPUT_PATH=""
NORMALIZE_OUTPUT_PATH="$DEFAULT_NORMALIZED_BACKUP_PATH"
NORMALIZE_ROLE="$(id -un)"
NORMALIZE_DATABASE="${PG_DATABASE:-testdb}"
NORMALIZE_ADMIN_ROLE="postgres"
NORMALIZE_PORT="${PG_PORT:-5432}"
NORMALIZE_PRELOAD_LIBRARIES="${PG_SHARED_PRELOAD_LIBRARIES:-pg_cron}"
NORMALIZE_DATABASE_EXPLICIT=0
while [ $# -gt 0 ]; do
case $1 in
-h|--help)
help_normalize_backup
exit 0
;;
-o|--output)
if [ $# -lt 2 ]; then
log error "normalize-backup: --output requires a path"
exit 3
fi
NORMALIZE_OUTPUT_PATH="$2"
shift 2
;;
--role)
if [ $# -lt 2 ]; then
log error "normalize-backup: --role requires a name"
exit 3
fi
NORMALIZE_ROLE="$2"
shift 2
;;
--database)
if [ $# -lt 2 ]; then
log error "normalize-backup: --database requires a name"
exit 3
fi
NORMALIZE_DATABASE="$2"
NORMALIZE_DATABASE_EXPLICIT=1
shift 2
;;
--admin-role)
if [ $# -lt 2 ]; then
log error "normalize-backup: --admin-role requires a name"
exit 3
fi
NORMALIZE_ADMIN_ROLE="$2"
shift 2
;;
--*|-*)
log error "normalize-backup argument $1 does not exist"
exit 9
;;
*)
if [ -n "$NORMALIZE_INPUT_PATH" ]; then
log error "normalize-backup: unexpected argument $1"
exit 9
fi
NORMALIZE_INPUT_PATH="$1"
shift
;;
esac
done
if [ -z "$NORMALIZE_INPUT_PATH" ]; then
NORMALIZE_INPUT_PATH="$DEFAULT_BACKUP_PATH"
fi
if [ -z "$NORMALIZE_ROLE" ] || [ -z "$NORMALIZE_DATABASE" ] || [ -z "$NORMALIZE_ADMIN_ROLE" ]; then
log error "normalize-backup: role, database, and admin role must be non-empty"
exit 3
fi
if [ ! -d "$NORMALIZE_INPUT_PATH" ]; then
log error "normalize-backup: input backup directory not found: $NORMALIZE_INPUT_PATH"
exit 3
fi
if [ ! -f "$NORMALIZE_INPUT_PATH/base.tar.gz" ]; then
log error "normalize-backup: missing $NORMALIZE_INPUT_PATH/base.tar.gz"
exit 3
fi
NORMALIZE_INPUT_ABS=$(realpath "$NORMALIZE_INPUT_PATH")
NORMALIZE_OUTPUT_ABS=$(realpath -m "$NORMALIZE_OUTPUT_PATH")
if [ "$NORMALIZE_INPUT_ABS" = "$NORMALIZE_OUTPUT_ABS" ]; then
log error "normalize-backup: input and output paths must differ"
exit 1
fi
case "$NORMALIZE_OUTPUT_ABS/" in
"$NORMALIZE_INPUT_ABS"/*)
log error "normalize-backup: output path must not be inside input backup"
exit 1
;;
esac
case "$NORMALIZE_INPUT_ABS/" in
"$NORMALIZE_OUTPUT_ABS"/*)
log error "normalize-backup: input backup must not be inside output path"
exit 1
;;
esac
mkdir -p "$(dirname "$NORMALIZE_OUTPUT_ABS")"
NORMALIZE_TMPDIR=$(mktemp -d)
NORMALIZE_PGDATA="$NORMALIZE_TMPDIR/data"
NORMALIZE_SOCKDIR="$NORMALIZE_TMPDIR/sock"
NORMALIZE_LOG="$NORMALIZE_TMPDIR/postgres.log"
trap '___normalize_backup_cleanup "$NORMALIZE_TMPDIR" "$NORMALIZE_PGDATA"' EXIT INT HUP
mkdir -m 700 "$NORMALIZE_PGDATA"
mkdir -p "$NORMALIZE_SOCKDIR" "$NORMALIZE_PGDATA/pg_wal"
log notice "extracting backup into temporary PGDATA"
tar -xzf "$NORMALIZE_INPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA"
if [ -f "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" ]; then
tar -xzf "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" -C "$NORMALIZE_PGDATA/pg_wal"
fi
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
if [ ! -f "$NORMALIZE_PGDATA/postgresql.conf" ]; then
cat > "$NORMALIZE_PGDATA/postgresql.conf" <<EOF
listen_addresses = ''
port = $NORMALIZE_PORT
unix_socket_directories = '$NORMALIZE_SOCKDIR'
logging_collector = off
shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
cron.database_name = '$NORMALIZE_DATABASE'
cron.host = '$NORMALIZE_SOCKDIR'
EOF
fi
cat > "$NORMALIZE_PGDATA/pg_hba.conf" <<EOF
local all all trust
EOF
: > "$NORMALIZE_PGDATA/pg_ident.conf"
cat > "$NORMALIZE_PGDATA/postgresql.auto.conf" <<EOF
# Local-dev normalized backup overrides.
listen_addresses = ''
port = '$NORMALIZE_PORT'
unix_socket_directories = '$NORMALIZE_SOCKDIR'
logging_collector = 'off'
hba_file = '$NORMALIZE_PGDATA/pg_hba.conf'
ident_file = '$NORMALIZE_PGDATA/pg_ident.conf'
shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
cron.database_name = '$NORMALIZE_DATABASE'
cron.host = '$NORMALIZE_SOCKDIR'
EOF
log notice "starting temporary local cluster"
with_closed_fds pg_ctl -D "$NORMALIZE_PGDATA" -o "-F" -w start > "$NORMALIZE_LOG" 2>&1 || {
log error "normalize-backup: failed to start temporary cluster; see $NORMALIZE_LOG"
exit 1
}
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" postgres
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template1
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template0
if [ "$NORMALIZE_DATABASE_EXPLICIT" -eq 0 ]; then
detected_database=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
-tAc "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate AND datname <> 'postgres' ORDER BY oid LIMIT 1" 2>/dev/null | sed '/^$/d' | head -n 1)
if [ -n "$detected_database" ]; then
NORMALIZE_DATABASE="$detected_database"
fi
fi
NORMALIZE_ROLE_SQL=$(___sql_literal "$NORMALIZE_ROLE")
psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
-v ON_ERROR_STOP=1 -c "DO \$\$ DECLARE v_role text := $NORMALIZE_ROLE_SQL; BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = v_role) THEN EXECUTE format('CREATE ROLE %I LOGIN SUPERUSER', v_role); ELSE EXECUTE format('ALTER ROLE %I LOGIN SUPERUSER', v_role); END IF; END \$\$;"
NORMALIZE_DATABASE_SQL=$(___sql_literal "$NORMALIZE_DATABASE")
NORMALIZE_DATABASE_EXISTS=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
-tAc "SELECT 1 FROM pg_database WHERE datname = $NORMALIZE_DATABASE_SQL" 2>/dev/null || true)
if [ "$NORMALIZE_DATABASE_EXISTS" != "1" ]; then
createdb -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" \
-O "$NORMALIZE_ROLE" "$NORMALIZE_DATABASE"
fi
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" "$NORMALIZE_DATABASE"
log notice "stopping temporary local cluster"
pg_ctl -D "$NORMALIZE_PGDATA" -m fast -w stop >> "$NORMALIZE_LOG" 2>&1
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
log notice "writing normalized backup to $WHITE$NORMALIZE_OUTPUT_ABS$NC"
rm -rf "$NORMALIZE_OUTPUT_ABS"
mkdir -p "$NORMALIZE_OUTPUT_ABS"
tar -czf "$NORMALIZE_OUTPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA" .
printf '%s\n' "$NORMALIZE_DATABASE" > "$NORMALIZE_OUTPUT_ABS/database_name"
___normalize_backup_cleanup "$NORMALIZE_TMPDIR" ""
trap - EXIT INT HUP
restore_namespace
}
subcommand_log() { subcommand_log() {
change_namespace 'db log' change_namespace 'db log'
: "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}" : "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}"
@@ -1175,6 +1456,7 @@ ___diff_dump_schema() {
local port="$2" local port="$2"
local output_file="$3" local output_file="$3"
local tables="${4:-}" local tables="${4:-}"
local database_name="${5:-${PG_DATABASE:-testdb}}"
log info "dumping schema to $WHITE$output_file$NC" log info "dumping schema to $WHITE$output_file$NC"
@@ -1189,17 +1471,17 @@ ___diff_dump_schema() {
IFS="$old_IFS" IFS="$old_IFS"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
pg_dump -h "$socket_dir" -p "$port" testdb \ pg_dump -h "$socket_dir" -p "$port" "$database_name" \
--schema-only --no-owner --no-privileges \ --schema-only --no-owner --no-privileges \
$table_args > "$output_file" 2>/dev/null $table_args > "$output_file" 2>/dev/null
# shellcheck disable=SC2086 # shellcheck disable=SC2086
pg_dump -h "$socket_dir" -p "$port" testdb \ pg_dump -h "$socket_dir" -p "$port" "$database_name" \
--data-only --no-owner --no-privileges \ --data-only --no-owner --no-privileges \
$table_args >> "$output_file" 2>/dev/null $table_args >> "$output_file" 2>/dev/null
else else
# Schema only # Schema only
pg_dump -h "$socket_dir" -p "$port" testdb \ pg_dump -h "$socket_dir" -p "$port" "$database_name" \
--schema-only --no-owner --no-privileges \ --schema-only --no-owner --no-privileges \
> "$output_file" 2>/dev/null > "$output_file" 2>/dev/null
fi fi
@@ -1208,7 +1490,8 @@ ___diff_dump_schema() {
___diff_immutable_tables() { ___diff_immutable_tables() {
local socket_dir="$1" local socket_dir="$1"
local port="$2" local port="$2"
psql -h "$socket_dir" -p "$port" -d testdb -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL' local database_name="${3:-${PG_DATABASE:-testdb}}"
psql -h "$socket_dir" -p "$port" -d "$database_name" -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
SELECT n.nspname || '.' || c.relname SELECT n.nspname || '.' || c.relname
FROM pg_inherits i FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_class c ON c.oid = i.inhrelid
@@ -1226,8 +1509,9 @@ ___diff_immutable_data() {
local sock2="$3" local sock2="$3"
local port2="$4" local port2="$4"
local out_file="$5" local out_file="$5"
local database_name="${6:-${PG_DATABASE:-testdb}}"
if ! psql -h "$sock1" -p "$port1" -d testdb -tAc \ if ! psql -h "$sock1" -p "$port1" -d "$database_name" -tAc \
"SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='hectic' AND c.relname='immutable';" \ "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='hectic' AND c.relname='immutable';" \
>/dev/null 2>&1 >/dev/null 2>&1
then then
@@ -1256,10 +1540,10 @@ ___diff_immutable_data() {
log info " $tbl" log info " $tbl"
: > "$data1" : > "$data1"
: > "$data2" : > "$data2"
pg_dump -h "$sock1" -p "$port1" testdb \ pg_dump -h "$sock1" -p "$port1" "$database_name" \
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \ --data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
> "$data1" 2>/dev/null || : > "$data1" 2>/dev/null || :
pg_dump -h "$sock2" -p "$port2" testdb \ pg_dump -h "$sock2" -p "$port2" "$database_name" \
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \ --data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
> "$data2" 2>/dev/null || : > "$data2" 2>/dev/null || :
{ {
@@ -1405,6 +1689,8 @@ subcommand_diff() {
DIFF_TABLES="" # TODO: add cron table DIFF_TABLES="" # TODO: add cron table
DIFF_NO_CRON=0 # TODO: useless option DIFF_NO_CRON=0 # TODO: useless option
DIFF_BACKUP_PATH="" DIFF_BACKUP_PATH=""
DIFF_IGNORE_MIGRATION_FAIL=0
DIFF_DATABASE="${PG_DATABASE:-testdb}"
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case $1 in case $1 in
@@ -1420,6 +1706,10 @@ subcommand_diff() {
DIFF_NO_CRON=1 DIFF_NO_CRON=1
shift shift
;; ;;
--ignore-migration-fail)
DIFF_IGNORE_MIGRATION_FAIL=1
shift
;;
-m|--mock) -m|--mock)
HYDRATE_USE_MOCK=1 HYDRATE_USE_MOCK=1
shift shift
@@ -1433,9 +1723,8 @@ subcommand_diff() {
exit 9 exit 9
;; ;;
*) *)
# NOTE: yes, RESTORE, not DIFF prefix if [ -z "$DIFF_BACKUP_PATH" ]; then
if [ -z "$RESTORE_BACKUP_PATH" ]; then DIFF_BACKUP_PATH="$1"
RESTORE_BACKUP_PATH="$1"
shift shift
else else
log error "diff: unexpected argument $1" log error "diff: unexpected argument $1"
@@ -1445,14 +1734,21 @@ subcommand_diff() {
esac esac
done done
if [ -z "$DIFF_BACKUP_PATH" ]; then
DIFF_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
fi
if [ -f "$DIFF_BACKUP_PATH/database_name" ]; then
DIFF_DATABASE=$(tr -d '\n' < "$DIFF_BACKUP_PATH/database_name")
fi
DIFF_TMPDIR="${LOCAL_DIR}/focus/database-diff-operation" DIFF_TMPDIR="${LOCAL_DIR}/focus/database-diff-operation"
DIFF_PGDATA1="$DIFF_TMPDIR/pgdata1" DIFF_PGDATA1="$DIFF_TMPDIR/pgdata1"
DIFF_PGDATA2="$DIFF_TMPDIR/pgdata2" DIFF_PGDATA2="$DIFF_TMPDIR/pgdata2"
# TODO: suka, logi drugie # TODO: suka, logi drugie
DIFF_PGLOGFILE1="$DIFF_PGDATA1/logfile" DIFF_PGLOGFILE1="$DIFF_PGDATA1/logfile"
DIFF_PGLOGFILE2="$DIFF_PGDATA2/logfile" DIFF_PGLOGFILE2="$DIFF_PGDATA2/logfile"
DIFF_PGURL1="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA1/sock" DIFF_PGURL1="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA1/sock"
DIFF_PGURL2="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA2/sock" DIFF_PGURL2="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA2/sock"
if [ "${DIFF_LOG+x}" ]; then if [ "${DIFF_LOG+x}" ]; then
log_pager -O "$DIFF_PGLOGFILE1" "$DIFF_PGLOGFILE2" log_pager -O "$DIFF_PGLOGFILE1" "$DIFF_PGLOGFILE2"
@@ -1474,21 +1770,26 @@ subcommand_diff() {
log notice "provisioning ${WHITE}DB1$NC (backup + migrations)" log notice "provisioning ${WHITE}DB1$NC (backup + migrations)"
PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore "$DIFF_BACKUP_PATH"
log info "applying migrations to ${WHITE}DB1$NC" log info "applying migrations to ${WHITE}DB1$NC"
subcommand_migrator migrate up all \ subcommand_migrator migrate up all \
--db-url \ --db-url \
"$DIFF_PGURL1" \ "$DIFF_PGURL1" \
|| { || {
log warn "migrations failed or none to apply" if [ "$DIFF_IGNORE_MIGRATION_FAIL" = "1" ]; then
log warn "migrations failed; continuing because --ignore-migration-fail is set"
else
log error "migrations failed"
exit 1
fi
} }
log notice "provisioning ${WHITE}DB2$NC (current sources)" log notice "provisioning ${WHITE}DB2$NC (current sources)"
log info "initializing ${WHITE}DB2$NC with postgres-init" log info "initializing ${WHITE}DB2$NC with postgres-init"
PG_WORKING_DIR="$DIFF_PGDATA2" \ PG_WORKING_DIR="$DIFF_PGDATA2" \
PG_DATABASE="testdb" \ PG_DATABASE="$DIFF_DATABASE" \
PG_DISABLE_LOGGING=1 \ PG_DISABLE_LOGGING=1 \
postgres-init || { postgres-init || {
log error "failed to initialize ${WHITE}DB2$NC" log error "failed to initialize ${WHITE}DB2$NC"
@@ -1514,8 +1815,11 @@ subcommand_diff() {
DIFF_DUMP1="$DIFF_TMPDIR/target.sql" DIFF_DUMP1="$DIFF_TMPDIR/target.sql"
DIFF_DUMP2="$DIFF_TMPDIR/source.sql" DIFF_DUMP2="$DIFF_TMPDIR/source.sql"
___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES" ___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES" "$DIFF_DATABASE"
___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES" ___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES" "$DIFF_DATABASE"
sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP1" || exit 1
sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP2" || exit 1
# Optional: filter out cron tables # Optional: filter out cron tables
if [ "$DIFF_NO_CRON" = "1" ]; then if [ "$DIFF_NO_CRON" = "1" ]; then
@@ -1541,7 +1845,8 @@ subcommand_diff() {
___diff_immutable_data \ ___diff_immutable_data \
"$DIFF_PGDATA1/sock" "5432" \ "$DIFF_PGDATA1/sock" "5432" \
"$DIFF_PGDATA2/sock" "5432" \ "$DIFF_PGDATA2/sock" "5432" \
"$DIFF_TMPDIR/diff" "$DIFF_TMPDIR/diff" \
"$DIFF_DATABASE"
data_status=$? data_status=$?
if [ "$schema_differs" = 0 ] && [ "$data_status" = 0 ]; then if [ "$schema_differs" = 0 ] && [ "$data_status" = 0 ]; then
@@ -1573,6 +1878,14 @@ if ! [ "${AS_LIBRARY+x}" ]; then
DB_URL=$2 DB_URL=$2
shift 2 shift 2
;; ;;
normalize-backup)
if [ "${SUBCOMMAND+x}" ]; then
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
else
SUBCOMMAND="normalize_backup"
fi
shift
;;
deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init) deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init)
if [ "${SUBCOMMAND+x}" ]; then if [ "${SUBCOMMAND+x}" ]; then
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")" REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
+222
View File
@@ -0,0 +1,222 @@
# shellcheck shell=dash
: "${SCRIPT_NAME:=$(basename "$0")}"
help() {
# shellcheck disable=SC2059
printf "$(cat <<EOF
${BGREEN}Usage:${NC} $SCRIPT_NAME [OPTIONS] <SUBCOMMAND> [OPTIONS]
PostgreSQL operations utility.
${BGREEN}Global Options:${NC}
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
${BGREEN}Subcommands:${NC}
${BCYAN}secrets load${NC} [OPTIONS] Apply hectic bundle and load secrets
${BGREEN}Environment:${NC}
${BBLACK}PGURL${NC} PostgreSQL connection string fallback
${BBLACK}DB_URL${NC} PostgreSQL connection string fallback
${BBLACK}HECTIC_DOTENV_CONTENT${NC} Raw dotenv content to load
${BBLACK}HECTIC_DOTENV_FILE${NC} Dotenv file to read when readable
${BBLACK}LOCAL_DIR${NC} Used with ENVIRONMENT for .env fallback
${BBLACK}ENVIRONMENT${NC} Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
EOF
)"
}
help_secrets_load() {
# shellcheck disable=SC2059
printf "$(cat <<EOF
${BGREEN}Usage:${NC} $SCRIPT_NAME ${BCYAN}secrets load${NC} [OPTIONS]
Apply the hectic SQL bundle and load dotenv-backed secrets into
${BBLACK}hectic.secret${NC}.
${BGREEN}Options:${NC}
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} <path> Dotenv file path
${BGREEN}Resolution order:${NC}
1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC}
2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC}
3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv
source can be resolved.
EOF
)"
}
resolve_pgurl() {
if [ -n "${PGURL:-}" ]; then
printf '%s' "$PGURL"
elif [ -n "${DB_URL:-}" ]; then
printf '%s' "$DB_URL"
else
return 1
fi
}
resolve_dotenv_content() {
dotenv_file="${1:-}"
if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then
printf '%s' "$HECTIC_DOTENV_CONTENT"
elif [ -n "$dotenv_file" ]; then
if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then
return 2
fi
cat "$dotenv_file"
elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
cat "${LOCAL_DIR}/.env.${ENVIRONMENT}"
else
return 1
fi
}
subcommand_secrets_load() {
change_namespace 'db ops secrets load'
dotenv_file="${HECTIC_DOTENV_FILE:-}"
while [ $# -gt 0 ]; do
case $1 in
-h|--help)
help_secrets_load
restore_namespace
exit 0
;;
-u|--url)
if [ $# -lt 2 ]; then
log error "missing value for $1"
restore_namespace
exit 3
fi
PGURL=$2
DB_URL=$2
shift 2
;;
-f|--dotenv-file)
if [ $# -lt 2 ]; then
log error "missing value for $1"
restore_namespace
exit 3
fi
dotenv_file=$2
shift 2
;;
--*|-*)
log error "secrets load argument $1 does not exist"
restore_namespace
exit 9
;;
*)
log error "secrets load subcommand $1 does not exist"
restore_namespace
exit 9
;;
esac
done
if ! pgurl="$(resolve_pgurl)"; then
log error "PGURL or DB_URL is required"
restore_namespace
exit 3
fi
if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then
:
else
dotenv_content_exit_code=$?
if [ "$dotenv_content_exit_code" -eq 2 ]; then
log error "dotenv file is not readable: $dotenv_file"
else
log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)"
fi
restore_namespace
exit 3
fi
log notice "apply hectic bundle and load secrets"
apply_hectic_bundle "$pgurl" "$dotenv_content"
restore_namespace
}
subcommand_secrets() {
change_namespace 'db ops secrets'
if [ $# -eq 0 ]; then
help_secrets_load
restore_namespace
exit 0
fi
subcommand=$1
shift
case $subcommand in
load)
subcommand_secrets_load "$@"
;;
-h|--help)
help_secrets_load
restore_namespace
exit 0
;;
*)
log error "secrets subcommand $subcommand does not exist"
restore_namespace
exit 9
;;
esac
}
while [ $# -gt 0 ]; do
case $1 in
-h|--help)
help
exit 0
;;
-u|--url)
if [ $# -lt 2 ]; then
log error "missing value for $1"
exit 3
fi
PGURL=$2
DB_URL=$2
shift 2
;;
--*|-*)
log error "argument $1 does not exist"
exit 9
;;
*)
break
;;
esac
done
subcommand="${1:-}"
if [ -z "$subcommand" ]; then
help
exit 0
fi
shift
case $subcommand in
secrets)
subcommand_secrets "$@"
;;
*)
log error "subcommand $subcommand does not exist"
exit 9
;;
esac
+31 -24
View File
@@ -1,32 +1,14 @@
{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }: { dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }:
let let
shell = "${dash}/bin/dash"; shell = "${dash}/bin/dash";
hecticInheritanceSqlPath = ../../lib/hook/sql/hectic-inheritance.sql;
hecticInheritance = runCommand "hectic-inheritance" { } '' hecticInheritance = runCommand "hectic-inheritance" { } ''
mkdir -p "$out/share/hectic" mkdir -p "$out/share/hectic"
cp ${hecticInheritanceSqlPath} "$out/share/hectic/hectic-inheritance.sql" cp ${self.lib.hectic.inheritance.path} "$out/share/hectic/hectic-inheritance.sql"
''; '';
# Materialize the templated version SQL into the Nix store as a real file applyBundle = self.lib.hectic.applyBundleScript;
# so it can be passed by path to psql -f (alongside the static siblings).
hecticVersionSqlFile = pkgs-writeText "hectic-version.sql" self.lib.hectic.version.sql;
pkgs-writeText = name: text: runCommand name { inherit text; passAsFile = [ "text" ]; } ''
cp "$textPath" "$out"
'';
hecticEnv = '' mkDbDev =
HECTIC_VERSION_SQL=${hecticVersionSqlFile}
HECTIC_SECRET_SQL=${self.lib.hectic.secret.path}
HECTIC_MIGRATION_SQL=${self.lib.hectic.migration.path}
HECTIC_INHERITANCE_SQL=${self.lib.hectic.inheritance.path}
export HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL
'';
applyBundle = builtins.readFile self.lib.hectic.applyBundleScript;
mkDatabase =
{ postgresql ? postgresql_17 }: { postgresql ? postgresql_17 }:
hectic.writeShellApplication { hectic.writeShellApplication {
inherit shell; inherit shell;
@@ -44,9 +26,8 @@ let
${builtins.readFile hectic.helpers.posix-shell.quote} ${builtins.readFile hectic.helpers.posix-shell.quote}
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat} ${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds} ${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
${hecticEnv}
${applyBundle} ${applyBundle}
${builtins.readFile ./database.sh} ${builtins.readFile ./db-dev.sh}
''; '';
meta = { meta = {
@@ -55,6 +36,31 @@ let
}; };
}; };
mkDbOps =
{ postgresql ? postgresql_17 }:
hectic.writeShellApplication {
inherit shell;
bashOptions = [
"errexit"
"nounset"
];
excludeShellChecks = [ "SC2209" ];
name = "db-ops";
runtimeInputs = [ postgresql coreutils ];
text = ''
${builtins.readFile hectic.helpers.posix-shell.log}
${builtins.readFile hectic.helpers.posix-shell.change_namespace}
${applyBundle}
${builtins.readFile ./db-ops.sh}
'';
meta = {
description = "PostgreSQL operations utility";
mainProgram = "db-ops";
};
};
mkPostgresInit = mkPostgresInit =
{ postgresql ? postgresql_17 }: { postgresql ? postgresql_17 }:
hectic.writeShellApplication { hectic.writeShellApplication {
@@ -92,7 +98,8 @@ let
}; };
in in
{ {
"db-tool" = lib.makeOverridable mkDatabase { }; "db-dev" = lib.makeOverridable mkDbDev { };
"db-ops" = lib.makeOverridable mkDbOps { };
"postgres-init" = lib.makeOverridable mkPostgresInit { }; "postgres-init" = lib.makeOverridable mkPostgresInit { };
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { }; "postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
"hectic-inheritance" = hecticInheritance; "hectic-inheritance" = hecticInheritance;
+37
View File
@@ -1,9 +1,46 @@
#!/bin/dash #!/bin/dash
# Stop a local PostgreSQL cluster started by postgres-init.
#
# Public contract:
# - Accepts PG_WORKING_DIR directly, or derives it from LOCAL_DIR.
# - If no cluster root can be resolved, exits successfully without doing
# anything. This keeps cleanup safe in traps and partial-init flows.
# - If no postmaster.pid exists, treats the cluster as already stopped.
# - NO_TTY=1 redirects pg_ctl output into a temp log file instead of writing to
# the current terminal.
postgres_cleanup_help() {
cat <<'EOF'
Usage: postgres-cleanup
Stop a local PostgreSQL cluster if it is running.
Environment:
PG_WORKING_DIR Cluster root directory
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
NO_TTY Redirect pg_ctl output to a temp file
Behavior:
- Returns success if the cluster directory cannot be resolved.
- Returns success if postmaster.pid is already absent.
- Ignores pg_ctl stop errors so cleanup remains trap-safe.
EOF
}
postgres_cleanup_main() { postgres_cleanup_main() {
case "${1:-}" in
-h|--help)
postgres_cleanup_help
return 0
;;
esac
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}" : "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
# Cleanup is intentionally forgiving so it can be used in traps after
# partial startup failures without masking the real error.
if [ "${NO_TTY:-0}" = "1" ]; then if [ "${NO_TTY:-0}" = "1" ]; then
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)" _pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || : pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
+181 -5
View File
@@ -1,6 +1,58 @@
#!/bin/dash #!/bin/dash
# Initialize or reuse a local PostgreSQL cluster for development workflows.
#
# Public contract:
# - Requires either PG_WORKING_DIR or LOCAL_DIR.
# - Produces a ready-to-use database and exports:
# POSTGRESQL_HOST, POSTGRESQL_PORT, POSTGRESQL_USER, POSTGRESQL_DATABASE,
# PGURL, and also the variable named by PG_URL_VAR.
# - Reuses an existing cluster only when PG_REUSE is set and PG_VERSION exists.
# - If a reused cluster cannot start, it is reinitialized automatically.
#
# Important knobs:
# - PG_WORKING_DIR: cluster root (defaults to $LOCAL_DIR/focus/postgresql)
# - PG_DATABASE: database to create/ensure (default: testdb)
# - PG_PORT: socket port (default: 5432)
# - PG_URL_VAR: alternate URL variable to export in addition to PGURL
# - PG_CONF_FILE: base postgresql.conf to copy on fresh init
# - PG_SHARED_PRELOAD_LIBRARIES: preload list; empty string disables pg_cron
# - PG_DISABLE_LOGGING=1: disable logging_collector in generated config
# - NO_TTY=1: redirect pg_ctl stop/start output into temp log files
postgres_init_help() {
cat <<'EOF'
Usage: postgres-init
Initialize or reuse a local PostgreSQL cluster.
Environment:
PG_WORKING_DIR Cluster root directory
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
PG_DATABASE Database name to create/ensure (default: testdb)
PG_PORT PostgreSQL port / unix socket port (default: 5432)
PG_URL_VAR Extra URL variable to export alongside PGURL
PG_CONF_FILE Base postgresql.conf copied on fresh init only
PG_SHARED_PRELOAD_LIBRARIES Preload list; empty disables pg_cron preload
PG_DISABLE_LOGGING Set to 1 to disable logging_collector
PG_REUSE Reuse existing cluster if PG_VERSION exists
NO_TTY Redirect pg_ctl output to temp files
Behavior:
- Rewrites runtime socket/port/cron settings on every launch.
- Creates the target database if missing.
- If a reused cluster exists but cannot start, reinitializes it automatically.
EOF
}
postgres_init_main() { postgres_init_main() {
case "${1:-}" in
-h|--help)
postgres_init_help
return 0
;;
esac
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2 printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
return 1 return 1
@@ -25,29 +77,149 @@ postgres_init_main() {
fi fi
mkdir -p "$sockdir" || return 1 mkdir -p "$sockdir" || return 1
# Reuse is optimistic: if a cluster exists on disk, try to start it first.
# We only destroy state after a real startup failure proves the cluster is
# broken, which keeps long-lived local DBs intact when possible.
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
if [ "$PG_REUSE" -eq 0 ]; then if [ "$PG_REUSE" -eq 0 ]; then
rm -rf "$data" "$sockdir" || return 1 rm -rf "$data" "$sockdir" || return 1
mkdir -p "$sockdir" || return 1 mkdir -p "$sockdir" || return 1
initdb -D "$data" --no-locale -E UTF8 || return 1 initdb -D "$data" --no-locale -E UTF8 || return 1
if [ -n "${PG_CONF_FILE:-}" ]; then if [ -n "${PG_CONF_FILE:-}" ]; then
# PG_CONF_FILE replaces the base postgresql.conf on fresh init only. We
# still append runtime values later so the helper can relocate sockets,
# ports, and cron settings regardless of the custom file contents.
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; } [ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1 cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
else else
{ printf '%s\n' "listen_addresses = ''"; [ "$PG_DISABLE_LOGGING" -eq 0 ] && { printf '%s\n' 'logging_collector = on'; printf '%s\n' "log_directory = 'log'"; }; [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ] && { printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"; printf '%s\n' "cron.database_name = '$db'"; printf '%s\n' "cron.host = '$sockdir'"; }; :; } >> "$data/postgresql.conf" || return 1 {
printf '%s\n' "listen_addresses = ''"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' 'logging_collector = on'
printf '%s\n' "log_directory = 'log'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.conf" || return 1
fi fi
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1 sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
fi fi
# Rewrite the runtime knobs on every launch. Reused clusters may have been
# started from another workspace/session, so we must override stale socket,
# port, and pg_cron wiring before attempting startup.
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1 sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1 sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1 { printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
{
printf '%s\n' "port = '$PG_PORT'"
printf '%s\n' "unix_socket_directories = '$sockdir'"
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' "logging_collector = 'on'"
else
printf '%s\n' "logging_collector = 'off'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.auto.conf" || return 1
_pg_start_exit=0
if [ "${NO_TTY:-0}" = "1" ]; then if [ "${NO_TTY:-0}" = "1" ]; then
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)" _pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || return 2 with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2 printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
else else
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || return 2 with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
fi
if [ "$_pg_start_exit" -ne 0 ]; then
if [ "$PG_REUSE" -eq 1 ]; then
# Self-heal path: the on-disk cluster exists but cannot complete startup
# (for example bad WAL / invalid checkpoint). At this point preserving
# the broken state is less useful than reinitializing automatically.
printf '%s\n' "postgres-init: reused cluster failed to start; reinitializing $wd" >&2
PG_REUSE=0
rm -rf "$data" "$sockdir" || return 1
mkdir -p "$sockdir" || return 1
initdb -D "$data" --no-locale -E UTF8 || return 1
if [ -n "${PG_CONF_FILE:-}" ]; then
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
else
{
printf '%s\n' "listen_addresses = ''"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' 'logging_collector = on'
printf '%s\n' "log_directory = 'log'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.conf" || return 1
fi
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
{
printf '%s\n' "port = '$PG_PORT'"
printf '%s\n' "unix_socket_directories = '$sockdir'"
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
printf '%s\n' "logging_collector = 'on'"
else
printf '%s\n' "logging_collector = 'off'"
fi
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
printf '%s\n' "cron.database_name = '$db'"
printf '%s\n' "cron.host = '$sockdir'"
fi
} >> "$data/postgresql.auto.conf" || return 1
_pg_start_exit=0
if [ "${NO_TTY:-0}" = "1" ]; then
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
else
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
fi
[ "$_pg_start_exit" -eq 0 ] || return 2
printf '%s\n' "postgres-init: reinitialized broken cluster at $wd" >&2
else
return 2
fi
fi fi
user="$(id -un)" || return 1 user="$(id -un)" || return 1
@@ -61,9 +233,13 @@ postgres_init_main() {
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1 psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db" export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
_pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}" # Export both names for compatibility: some callers consume plain PGURL,
# while multi-cluster shells also need a project-specific variable like
# WIPE_PGURL or BMSEARCH_PGURL in the same environment.
PGURL="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
export PGURL
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
export "${PG_URL_VAR}=${_pg_url}" || return 1 export "${PG_URL_VAR}=${PGURL}" || return 1
return 0 return 0
} }
+8 -1
View File
@@ -123,6 +123,7 @@ in {
py3-swifter = pkgs.callPackage ./py3-swifter.nix {}; py3-swifter = pkgs.callPackage ./py3-swifter.nix {};
py3-aiogram-newsletter = pkgs.callPackage ./py3-aiogram-newsletter.nix {}; py3-aiogram-newsletter = pkgs.callPackage ./py3-aiogram-newsletter.nix {};
py3-openai-shap-e = pkgs.callPackage ./py3-openai-shap-e.nix {}; py3-openai-shap-e = pkgs.callPackage ./py3-openai-shap-e.nix {};
hiddify-core = pkgs.callPackage ./hiddify-core {};
nvim-alias = pkgs.callPackage ./nvim-alias.nix {}; nvim-alias = pkgs.callPackage ./nvim-alias.nix {};
bolt-unpack = pkgs.callPackage ./bolt-unpack.nix {}; bolt-unpack = pkgs.callPackage ./bolt-unpack.nix {};
merge-archive = pkgs.callPackage ./merge-archive {}; merge-archive = pkgs.callPackage ./merge-archive {};
@@ -131,6 +132,7 @@ in {
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {}; github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {}; supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
migration-name = pkgs.callPackage ./migration-name.nix {}; migration-name = pkgs.callPackage ./migration-name.nix {};
plgo = pkgs.callPackage ./plgo {};
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs; pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs; pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs; pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
@@ -141,14 +143,18 @@ in {
watch = pkgs.callPackage ./c/watch/default.nix {}; watch = pkgs.callPackage ./c/watch/default.nix {};
support-bot = pkgs.callPackage ./support-bot {}; support-bot = pkgs.callPackage ./support-bot {};
gitea-heatmap = pkgs.callPackage ./gitea {}; gitea-heatmap = pkgs.callPackage ./gitea {};
gitea-runner-nix-image = pkgs.callPackage ./gitea-runner-nix-image {};
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {}; nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {}; "sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
deploy = pkgs.callPackage ./deploy { inherit inputs; }; deploy = pkgs.callPackage ./deploy { inherit inputs; };
element-web = pkgs.callPackage ./element-web {};
shellplot = pkgs.callPackage ./shellplot {}; shellplot = pkgs.callPackage ./shellplot {};
which-country-rs = pkgs.callPackage ./which-country-rs {};
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {}; onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
migrator = pkgs.callPackage ./migrator { inherit self; }; migrator = pkgs.callPackage ./migrator { inherit self; };
"parse-uri" = pkgs.callPackage ./parse-uri {}; "parse-uri" = pkgs.callPackage ./parse-uri {};
"db-tool" = dbToolPkgs."db-tool"; "db-dev" = dbToolPkgs."db-dev";
"db-ops" = dbToolPkgs."db-ops";
"postgres-init" = dbToolPkgs."postgres-init"; "postgres-init" = dbToolPkgs."postgres-init";
"postgres-cleanup" = dbToolPkgs."postgres-cleanup"; "postgres-cleanup" = dbToolPkgs."postgres-cleanup";
"hectic-inheritance" = dbToolPkgs."hectic-inheritance"; "hectic-inheritance" = dbToolPkgs."hectic-inheritance";
@@ -170,5 +176,6 @@ in {
pg-15-ext-smtp-client = buildSmtpExt pkgs "15"; pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15"; pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
pg-15-ext-plsh = buildPlShExt pkgs "15"; pg-15-ext-plsh = buildPlShExt pkgs "15";
stable-video-diffusion-api = pkgs.callPackage ./stable-video-diffusion-api {};
media-browser = pkgs.callPackage ./media-browser {}; media-browser = pkgs.callPackage ./media-browser {};
} }
+114
View File
@@ -0,0 +1,114 @@
{
lib,
stdenv,
fetchFromGitHub,
jq,
nodejs,
jitsi-meet,
fetchPnpmDeps,
pnpm_10,
pnpmConfigHook,
faketty,
config,
conf ? config.element-web.conf or { },
}:
let
pnpm = pnpm_10;
patchDir = ./patches;
patches = if builtins.pathExists patchDir then map (name: patchDir + "/${name}") (
builtins.filter (name: lib.hasSuffix ".patch" name) (builtins.attrNames (builtins.readDir patchDir))
) else [ ];
noPhoningHome = {
disable_guests = true;
};
jitsi-meet-override = jitsi-meet.overrideAttrs (previousAttrs: {
meta = removeAttrs previousAttrs.meta [ "knownVulnerabilities" ];
});
element-web-unwrapped = stdenv.mkDerivation (finalAttrs: {
pname = "element-web";
version = "1.12.20";
src = fetchFromGitHub {
owner = "element-hq";
repo = "element-web";
tag = "v${finalAttrs.version}";
hash = "sha256-pbzuPgKJ0DmrDSTO7ZTDArX+Xr9k/ndAGZvQg2kMTMQ=";
};
#inherit patches; #WIP
pnpmDeps = fetchPnpmDeps {
pname = "element";
inherit (finalAttrs) version src;
inherit pnpm;
fetcherVersion = 3;
hash = "sha256-snm7vaHCVX6vYrkmsz5HlYMKx5Ks3K9jvUinkJ41CU0=";
};
nativeBuildInputs = [
jq
nodejs
pnpm
pnpmConfigHook
faketty
];
buildPhase = ''
runHook preBuild
cd apps/web
export VERSION=${finalAttrs.version}
faketty pnpm run build
runHook postBuild
'';
installPhase = ''
runHook preInstall
cp -R webapp $out
cp ${jitsi-meet-override}/libs/external_api.min.js $out/jitsi_external_api.min.js
echo "${finalAttrs.version}" > "$out/version"
jq -s '.[0] * $conf' "config.sample.json" --argjson "conf" '${builtins.toJSON noPhoningHome}' > "$out/config.json"
runHook postInstall
'';
meta = {
description = "Glossy Matrix collaboration client for the web";
homepage = "https://element.io/";
changelog = "https://github.com/element-hq/element-web/blob/v${finalAttrs.version}/CHANGELOG.md";
teams = [ lib.teams.matrix ];
license = lib.licenses.agpl3Plus;
platforms = lib.platforms.all;
};
});
in
if conf == { } then
element-web-unwrapped
else
stdenv.mkDerivation {
pname = "${element-web-unwrapped.pname}-wrapped";
inherit (element-web-unwrapped) version meta;
dontUnpack = true;
nativeBuildInputs = [ jq ];
installPhase = ''
runHook preInstall
mkdir -p $out
ln -s ${element-web-unwrapped}/* $out
rm $out/config.json
jq -s '.[0] * $conf' "${element-web-unwrapped}/config.json" --argjson "conf" ${lib.escapeShellArg (builtins.toJSON conf)} > "$out/config.json"
runHook postInstall
'';
passthru = {
inherit conf;
};
}
@@ -0,0 +1,481 @@
diff --git a/apps/web/src/IConfigOptions.ts b/apps/web/src/IConfigOptions.ts
index a3a6a32..c61c24f 100644
--- a/apps/web/src/IConfigOptions.ts
+++ b/apps/web/src/IConfigOptions.ts
@@ -105,6 +105,15 @@ export interface IConfigOptions {
show_labs_settings: boolean;
features?: Record<string, boolean>; // <FeatureName, EnabledBool>
+ hectic?: {
+ // This local package config intentionally uses the documented camelCase path
+ // `hectic.videoMessages.enabled`.
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ videoMessages?: {
+ enabled?: boolean;
+ };
+ };
+
/**
* Bug report endpoint URL. "local" means the logs should not be uploaded.
*/
diff --git a/apps/web/src/SdkConfig.ts b/apps/web/src/SdkConfig.ts
index 4be6464..9f48743 100644
--- a/apps/web/src/SdkConfig.ts
+++ b/apps/web/src/SdkConfig.ts
@@ -28,6 +28,12 @@ export const DEFAULTS: DeepReadonly<IConfigOptions> = {
integrations_ui_url: "https://scalar.vector.im/",
integrations_rest_url: "https://scalar.vector.im/api",
show_labs_settings: false,
+ hectic: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ videoMessages: {
+ enabled: false,
+ },
+ },
force_verification: false,
jitsi: {
diff --git a/apps/web/src/components/views/rooms/MessageComposer.tsx b/apps/web/src/components/views/rooms/MessageComposer.tsx
index 06c843f..eea76b7 100644
--- a/apps/web/src/components/views/rooms/MessageComposer.tsx
+++ b/apps/web/src/components/views/rooms/MessageComposer.tsx
@@ -33,6 +33,7 @@ import ReplyPreview from "./ReplyPreview";
import { UserIdentityWarning } from "./UserIdentityWarning";
import { UPDATE_EVENT } from "../../../stores/AsyncStore";
import VoiceRecordComposerTile from "./VoiceRecordComposerTile";
+import VideoRecordComposerTile from "./VideoRecordComposerTile";
import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore";
import { RecordingState } from "../../../audio/VoiceRecording";
import type ResizeNotifier from "../../../utils/ResizeNotifier";
@@ -92,6 +93,7 @@ interface IState {
composerContent: string;
isComposerEmpty: boolean;
haveRecording: boolean;
+ haveVideoRecording: boolean;
recordingTimeLeftSeconds?: number;
me?: RoomMember;
isMenuOpen: boolean;
@@ -113,6 +115,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
private dispatcherRef?: string;
private messageComposerInput = createRef<SendMessageComposerClass>();
private voiceRecordingButton = createRef<VoiceRecordComposerTile>();
+ private videoRecordingButton = createRef<VideoRecordComposerTile>();
private ref = createRef<HTMLDivElement>();
private instanceId: number;
@@ -144,6 +147,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
isComposerEmpty: initialComposerContent?.length === 0,
composerContent: initialComposerContent,
haveRecording: false,
+ haveVideoRecording: false,
recordingTimeLeftSeconds: undefined, // when set to a number, shows a toast
isMenuOpen: false,
isStickerPickerOpen: false,
@@ -526,6 +530,18 @@ export class MessageComposer extends React.Component<IProps, IState> {
}
};
+ private onVideoRecordStartClick = (): void => {
+ this.videoRecordingButton.current?.onRecordStartEndClick();
+
+ if (this.context.narrow) {
+ this.toggleButtonMenu();
+ }
+ };
+
+ private onVideoRecordingStateChange = (haveVideoRecording: boolean): void => {
+ this.setState({ haveVideoRecording });
+ };
+
public render(): React.ReactNode {
let leftIcon: false | JSX.Element = false;
if (!this.state.isWysiwygLabEnabled) {
@@ -561,13 +577,14 @@ export class MessageComposer extends React.Component<IProps, IState> {
const menuPosition = this.getMenuPosition();
const canSendMessages = this.context.canSendMessages && !this.context.tombstone;
+ const hasActiveRecording = this.state.haveRecording || this.state.haveVideoRecording;
let composer: ReactNode;
if (canSendMessages) {
if (this.state.isWysiwygLabEnabled && menuPosition) {
composer = (
<SendWysiwygComposer
key="controls_input"
- disabled={this.state.haveRecording}
+ disabled={hasActiveRecording}
onChange={this.onWysiwygChange}
onSend={this.sendMessage}
isRichTextEnabled={this.state.isRichTextEnabled}
@@ -588,7 +605,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
relation={this.props.relation}
replyToEvent={this.props.replyToEvent}
onChange={this.onChange}
- disabled={this.state.haveRecording}
+ disabled={hasActiveRecording}
toggleStickerPickerOpen={this.toggleStickerPickerOpen}
/>
);
@@ -608,6 +625,11 @@ export class MessageComposer extends React.Component<IProps, IState> {
replyToEvent={this.props.replyToEvent}
/>
</Tooltip>,
+ <VideoRecordComposerTile
+ key="controls_video_record"
+ ref={this.videoRecordingButton}
+ onRecordingStateChange={this.onVideoRecordingStateChange}
+ />,
);
} else if (this.context.tombstone) {
const replacementRoomId = this.context.tombstone.getContent()["replacement_room"];
@@ -688,12 +710,13 @@ export class MessageComposer extends React.Component<IProps, IState> {
{canSendMessages && (
<MessageComposerButtons
addEmoji={this.addEmoji}
- haveRecording={this.state.haveRecording}
+ haveRecording={hasActiveRecording}
isMenuOpen={this.state.isMenuOpen}
isStickerPickerOpen={this.state.isStickerPickerOpen}
menuPosition={menuPosition}
relation={this.props.relation}
onRecordStartEndClick={this.onRecordStartEndClick}
+ onVideoRecordStartClick={this.onVideoRecordStartClick}
setStickerPickerOpen={this.setStickerPickerOpen}
showLocationButton={
!window.electron && SettingsStore.getValue(UIFeature.LocationSharing)
diff --git a/apps/web/src/components/views/rooms/MessageComposerButtons.tsx b/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
index 6f4f5d1..0c1b7ff 100644
--- a/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
+++ b/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
@@ -44,6 +44,8 @@ import { useSettingValue } from "../../../hooks/useSettings";
import AccessibleButton, { type ButtonEvent } from "../elements/AccessibleButton";
import { useScopedRoomContext } from "../../../contexts/ScopedRoomContext.tsx";
import { useRoomUploadViewModel } from "../../../viewmodels/room/RoomUploadViewModel.tsx";
+import SdkConfig from "../../../SdkConfig";
+import { isVideoMessageRecorderSupported } from "./VideoRecordComposerTile";
interface IProps {
addEmoji: (emoji: string) => boolean;
@@ -52,6 +54,7 @@ interface IProps {
isStickerPickerOpen: boolean;
menuPosition?: MenuProps;
onRecordStartEndClick: () => void;
+ onVideoRecordStartClick: () => void;
relation?: IEventRelation;
setStickerPickerOpen: (isStickerPickerOpen: boolean) => void;
showLocationButton: boolean;
@@ -103,6 +106,7 @@ const MessageComposerButtons: React.FC<IProps> = (props: IProps) => {
)),
showStickersButton(props),
voiceRecordingButton(props, narrow),
+ videoRecordingButton(props, narrow),
props.showPollsButton ? pollButton(room, props.relation) : null,
showLocationButton(props, room, matrixClient),
];
@@ -122,6 +126,7 @@ const MessageComposerButtons: React.FC<IProps> = (props: IProps) => {
moreButtons = [
showStickersButton(props),
voiceRecordingButton(props, narrow),
+ videoRecordingButton(props, narrow),
props.showPollsButton ? pollButton(room, props.relation) : null,
showLocationButton(props, room, matrixClient),
];
@@ -203,6 +208,25 @@ function voiceRecordingButton(props: IProps, narrow: boolean): ReactElement | nu
);
}
+function videoRecordingButton(props: IProps, narrow: boolean): ReactElement | null {
+ // The current recorder skeleton follows the voice recorder and stays out of narrow mode.
+ if (narrow || SdkConfig.get("hectic")?.videoMessages?.enabled !== true || !isVideoMessageRecorderSupported()) {
+ return null;
+ }
+
+ return (
+ <CollapsibleButton
+ key="video_message_send"
+ className="mx_MessageComposer_button"
+ data-testid="video-message-button"
+ onClick={props.onVideoRecordStartClick}
+ title={_t("composer|video_message_button")}
+ >
+ <span aria-hidden="true">●</span>
+ </CollapsibleButton>
+ );
+}
+
function pollButton(room: Room, relation?: IEventRelation): ReactElement {
return <PollButton key="polls" room={room} relation={relation} />;
}
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
new file mode 100644
index 0000000..c5f0adc
--- /dev/null
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
@@ -0,0 +1,249 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import React, { type ReactNode } from "react";
+import { logger } from "matrix-js-sdk/src/logger";
+import { DeleteIcon, SendSolidIcon, StopSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
+
+import { _t } from "../../../languageHandler";
+import AccessibleButton from "../elements/AccessibleButton";
+
+interface IProps {
+ onRecordingStateChange: (haveVideoRecording: boolean) => void;
+}
+
+interface IState {
+ error?: string;
+ isRecording: boolean;
+ previewUrl?: string;
+}
+
+export function isVideoMessageRecorderSupported(): boolean {
+ return (
+ typeof window !== "undefined" &&
+ typeof MediaRecorder !== "undefined" &&
+ typeof navigator !== "undefined" &&
+ !!navigator.mediaDevices?.getUserMedia
+ );
+}
+
+/**
+ * Container tile for rendering the config-gated video message recorder skeleton in the composer.
+ */
+export default class VideoRecordComposerTile extends React.PureComponent<IProps, IState> {
+ private chunks: Blob[] = [];
+ private mediaRecorder?: MediaRecorder;
+ private stream?: MediaStream;
+ private isRequestingMedia = false;
+ private isUnmounted = false;
+ private shouldDiscardRecording = false;
+
+ public constructor(props: IProps) {
+ super(props);
+
+ this.state = {
+ isRecording: false,
+ };
+ }
+
+ public componentWillUnmount(): void {
+ this.isUnmounted = true;
+ this.disposeRecording();
+ }
+
+ public onRecordStartEndClick = async (): Promise<void> => {
+ if (this.state.isRecording) {
+ this.stopRecording();
+ return;
+ }
+
+ await this.startRecording();
+ };
+
+ private startRecording = async (): Promise<void> => {
+ if (this.isRequestingMedia || this.mediaRecorder) return;
+
+ if (!isVideoMessageRecorderSupported()) {
+ if (!this.isUnmounted) {
+ this.setState({ error: _t("composer|video_message_error") });
+ this.props.onRecordingStateChange(false);
+ }
+ return;
+ }
+
+ this.isRequestingMedia = true;
+ this.shouldDiscardRecording = false;
+ this.revokePreviewUrl();
+ this.chunks = [];
+
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
+ if (this.isUnmounted) {
+ stream.getTracks().forEach((track) => track.stop());
+ return;
+ }
+
+ this.stream = stream;
+ const mediaRecorder = new MediaRecorder(stream);
+
+ mediaRecorder.ondataavailable = (ev: BlobEvent): void => {
+ if (ev.data.size > 0) {
+ this.chunks.push(ev.data);
+ }
+ };
+ mediaRecorder.onerror = (ev: Event): void => {
+ logger.error("Error recording video message:", ev);
+ this.setState({ error: _t("composer|video_message_error") });
+ };
+ mediaRecorder.onstop = this.onRecorderStop;
+
+ this.mediaRecorder = mediaRecorder;
+ mediaRecorder.start();
+
+ this.setState({ error: undefined, isRecording: true, previewUrl: undefined });
+ this.props.onRecordingStateChange(true);
+ } catch (e) {
+ logger.error("Error starting video message recording:", e);
+ this.stopStream();
+ if (!this.isUnmounted) {
+ this.setState({ error: _t("composer|video_message_error"), isRecording: false, previewUrl: undefined });
+ this.props.onRecordingStateChange(false);
+ }
+ } finally {
+ this.isRequestingMedia = false;
+ }
+ };
+
+ private stopRecording = (): void => {
+ if (!this.mediaRecorder) return;
+
+ if (this.mediaRecorder.state === "inactive") {
+ this.onRecorderStop();
+ } else {
+ this.mediaRecorder.stop();
+ }
+ };
+
+ private onRecorderStop = (): void => {
+ const shouldDiscardRecording = this.shouldDiscardRecording;
+ const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
+
+ this.mediaRecorder = undefined;
+ this.stopStream();
+
+ if (this.isUnmounted) {
+ this.chunks = [];
+ return;
+ }
+
+ if (!hasRecording) {
+ this.chunks = [];
+ this.setState({ isRecording: false, previewUrl: undefined });
+ this.props.onRecordingStateChange(false);
+ return;
+ }
+
+ const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "video/webm" });
+ const previewUrl = URL.createObjectURL(blob);
+
+ this.chunks = [];
+ this.revokePreviewUrl();
+ this.setState({ isRecording: false, previewUrl });
+ this.props.onRecordingStateChange(true);
+ };
+
+ private onCancel = (): void => {
+ this.shouldDiscardRecording = true;
+ this.disposeRecording();
+ this.setState({ error: undefined, isRecording: false, previewUrl: undefined });
+ this.props.onRecordingStateChange(false);
+ };
+
+ private disposeRecording(): void {
+ this.chunks = [];
+ this.stopStream();
+ this.revokePreviewUrl();
+
+ if (this.mediaRecorder) {
+ this.mediaRecorder.ondataavailable = null;
+ this.mediaRecorder.onerror = null;
+ this.mediaRecorder.onstop = null;
+
+ if (this.mediaRecorder.state !== "inactive") {
+ this.mediaRecorder.stop();
+ }
+ this.mediaRecorder = undefined;
+ }
+ }
+
+ private stopStream(): void {
+ this.stream?.getTracks().forEach((track) => track.stop());
+ this.stream = undefined;
+ }
+
+ private revokePreviewUrl(): void {
+ if (this.state.previewUrl) {
+ URL.revokeObjectURL(this.state.previewUrl);
+ }
+ }
+
+ public render(): ReactNode {
+ if (!this.state.isRecording && !this.state.previewUrl && !this.state.error) return null;
+
+ const stopButton = this.state.isRecording ? (
+ <AccessibleButton
+ className="mx_VoiceRecordComposerTile_stop"
+ data-testid="video-message-stop-button"
+ onClick={this.onRecordStartEndClick}
+ title={_t("composer|stop_video_message")}
+ >
+ <StopSolidIcon />
+ </AccessibleButton>
+ ) : null;
+
+ const sendButton = this.state.previewUrl ? (
+ <AccessibleButton
+ className="mx_VoiceRecordComposerTile_stop"
+ data-testid="video-message-send-button"
+ disabled={true}
+ onClick={null}
+ title={_t("composer|send_video_message")}
+ >
+ <SendSolidIcon />
+ </AccessibleButton>
+ ) : null;
+
+ return (
+ <>
+ {this.state.error && (
+ <span className="mx_VoiceRecordComposerTile_failedState" data-testid="video-message-error">
+ {this.state.error}
+ </span>
+ )}
+ <AccessibleButton
+ className="mx_VoiceRecordComposerTile_delete"
+ data-testid="video-message-cancel-button"
+ onClick={this.onCancel}
+ title={_t("composer|video_message_cancel")}
+ >
+ <DeleteIcon />
+ </AccessibleButton>
+ {stopButton}
+ {sendButton}
+ {this.state.previewUrl && (
+ <video
+ aria-label={_t("composer|video_message_preview")}
+ className="mx_VoiceMessagePrimaryContainer"
+ controls={true}
+ data-testid="video-message-preview"
+ src={this.state.previewUrl}
+ />
+ )}
+ </>
+ );
+ }
+}
diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json
index 4ed51db..e11fe90 100644
--- a/apps/web/src/i18n/strings/en_EN.json
+++ b/apps/web/src/i18n/strings/en_EN.json
@@ -641,8 +641,14 @@
"room_upgraded_notice": "This room has been replaced and is no longer active.",
"send_button_title": "Send message",
"send_button_voice_message": "Send voice message",
+ "send_video_message": "Send video message",
"send_voice_message": "Send voice message",
+ "stop_video_message": "Stop video recording",
"stop_voice_message": "Stop recording",
+ "video_message_button": "Record video message",
+ "video_message_cancel": "Cancel video message",
+ "video_message_error": "Unable to record video message",
+ "video_message_preview": "Video message preview",
"voice_message_button": "Voice Message"
},
"console_dev_note": "If you know what you're doing, Element is open-source, be sure to check out our GitHub (https://github.com/vector-im/element-web/) and contribute!",
@@ -0,0 +1,201 @@
diff --git a/apps/web/src/components/views/rooms/MessageComposer.tsx b/apps/web/src/components/views/rooms/MessageComposer.tsx
index eea76b7..3483c28 100644
--- a/apps/web/src/components/views/rooms/MessageComposer.tsx
+++ b/apps/web/src/components/views/rooms/MessageComposer.tsx
@@ -629,6 +629,9 @@ export class MessageComposer extends React.Component<IProps, IState> {
key="controls_video_record"
ref={this.videoRecordingButton}
onRecordingStateChange={this.onVideoRecordingStateChange}
+ relation={this.props.relation}
+ replyToEvent={this.props.replyToEvent}
+ room={this.props.room}
/>,
);
} else if (this.context.tombstone) {
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
index c5f0adc..851c64e 100644
--- a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
@@ -6,22 +6,39 @@ Please see LICENSE files in the repository root for full details.
*/
import React, { type ReactNode } from "react";
+import { type IEventRelation, type MatrixEvent, type Room } from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
import { DeleteIcon, SendSolidIcon, StopSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import { _t } from "../../../languageHandler";
+import { MatrixClientPeg } from "../../../MatrixClientPeg";
+import ContentMessages from "../../../ContentMessages";
import AccessibleButton from "../elements/AccessibleButton";
interface IProps {
onRecordingStateChange: (haveVideoRecording: boolean) => void;
+ relation?: IEventRelation;
+ replyToEvent?: MatrixEvent;
+ room: Room;
}
interface IState {
error?: string;
isRecording: boolean;
+ isSending: boolean;
previewUrl?: string;
}
+const PREFERRED_VIDEO_MIME_TYPES = ["video/webm;codecs=vp8,opus", "video/webm"];
+
+function preferredVideoMimeType(): string | undefined {
+ for (const mimeType of PREFERRED_VIDEO_MIME_TYPES) {
+ if (MediaRecorder.isTypeSupported(mimeType)) {
+ return mimeType;
+ }
+ }
+}
+
export function isVideoMessageRecorderSupported(): boolean {
return (
typeof window !== "undefined" &&
@@ -37,6 +54,7 @@ export function isVideoMessageRecorderSupported(): boolean {
export default class VideoRecordComposerTile extends React.PureComponent<IProps, IState> {
private chunks: Blob[] = [];
private mediaRecorder?: MediaRecorder;
+ private recordedFile?: File;
private stream?: MediaStream;
private isRequestingMedia = false;
private isUnmounted = false;
@@ -47,6 +65,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
this.state = {
isRecording: false,
+ isSending: false,
};
}
@@ -78,6 +97,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
this.isRequestingMedia = true;
this.shouldDiscardRecording = false;
this.revokePreviewUrl();
+ this.recordedFile = undefined;
this.chunks = [];
try {
@@ -88,7 +108,8 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
}
this.stream = stream;
- const mediaRecorder = new MediaRecorder(stream);
+ const mimeType = preferredVideoMimeType();
+ const mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
mediaRecorder.ondataavailable = (ev: BlobEvent): void => {
if (ev.data.size > 0) {
@@ -104,13 +125,18 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
this.mediaRecorder = mediaRecorder;
mediaRecorder.start();
- this.setState({ error: undefined, isRecording: true, previewUrl: undefined });
+ this.setState({ error: undefined, isRecording: true, isSending: false, previewUrl: undefined });
this.props.onRecordingStateChange(true);
} catch (e) {
logger.error("Error starting video message recording:", e);
this.stopStream();
if (!this.isUnmounted) {
- this.setState({ error: _t("composer|video_message_error"), isRecording: false, previewUrl: undefined });
+ this.setState({
+ error: _t("composer|video_message_error"),
+ isRecording: false,
+ isSending: false,
+ previewUrl: undefined,
+ });
this.props.onRecordingStateChange(false);
}
} finally {
@@ -129,6 +155,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
};
private onRecorderStop = (): void => {
+ const mimeType = this.mediaRecorder?.mimeType;
const shouldDiscardRecording = this.shouldDiscardRecording;
const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
@@ -142,29 +169,63 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
if (!hasRecording) {
this.chunks = [];
- this.setState({ isRecording: false, previewUrl: undefined });
+ this.recordedFile = undefined;
+ this.setState({ isRecording: false, isSending: false, previewUrl: undefined });
this.props.onRecordingStateChange(false);
return;
}
- const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "video/webm" });
- const previewUrl = URL.createObjectURL(blob);
+ const blob = new Blob(this.chunks, { type: mimeType || this.chunks[0]?.type || "video/webm" });
+ const file = new File([blob], `video-message-${Date.now()}.webm`, { type: blob.type || "video/webm" });
+ const previewUrl = URL.createObjectURL(file);
this.chunks = [];
+ this.recordedFile = file;
this.revokePreviewUrl();
- this.setState({ isRecording: false, previewUrl });
+ this.setState({ isRecording: false, isSending: false, previewUrl });
this.props.onRecordingStateChange(true);
};
+ private onSend = async (): Promise<void> => {
+ if (!this.recordedFile || this.state.isSending) return;
+
+ this.setState({ isSending: true });
+
+ try {
+ await ContentMessages.sharedInstance().sendContentToRoom(
+ this.recordedFile,
+ this.props.room.roomId,
+ this.props.relation,
+ MatrixClientPeg.safeGet(),
+ this.props.replyToEvent,
+ );
+ this.clearRecording();
+ this.props.onRecordingStateChange(false);
+ } catch (e) {
+ logger.error("Error sending video message recording:", e);
+ if (!this.isUnmounted) {
+ this.setState({ error: _t("composer|video_message_error"), isSending: false });
+ }
+ }
+ };
+
private onCancel = (): void => {
this.shouldDiscardRecording = true;
this.disposeRecording();
- this.setState({ error: undefined, isRecording: false, previewUrl: undefined });
+ this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
this.props.onRecordingStateChange(false);
};
+ private clearRecording(): void {
+ this.chunks = [];
+ this.recordedFile = undefined;
+ this.revokePreviewUrl();
+ this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
+ }
+
private disposeRecording(): void {
this.chunks = [];
+ this.recordedFile = undefined;
this.stopStream();
this.revokePreviewUrl();
@@ -209,8 +270,8 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
<AccessibleButton
className="mx_VoiceRecordComposerTile_stop"
data-testid="video-message-send-button"
- disabled={true}
- onClick={null}
+ disabled={!this.recordedFile || this.state.isSending}
+ onClick={this.onSend}
title={_t("composer|send_video_message")}
>
<SendSolidIcon />
@@ -0,0 +1,195 @@
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
index 851c64e..4285c72 100644
--- a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
@@ -30,6 +30,7 @@ interface IState {
}
const PREFERRED_VIDEO_MIME_TYPES = ["video/webm;codecs=vp8,opus", "video/webm"];
+const MAX_VIDEO_RECORDING_MS = 120000;
function preferredVideoMimeType(): string | undefined {
for (const mimeType of PREFERRED_VIDEO_MIME_TYPES) {
@@ -39,6 +40,18 @@ function preferredVideoMimeType(): string | undefined {
}
}
+function isPermissionDeniedError(error: unknown): boolean {
+ if (error instanceof DOMException) {
+ return error.name === "NotAllowedError" || error.name === "PermissionDeniedError";
+ }
+
+ if (error instanceof Error) {
+ return /permission denied|not allowed|denied permission/i.test(error.message);
+ }
+
+ return false;
+}
+
export function isVideoMessageRecorderSupported(): boolean {
return (
typeof window !== "undefined" &&
@@ -56,6 +69,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
private mediaRecorder?: MediaRecorder;
private recordedFile?: File;
private stream?: MediaStream;
+ private durationCapTimer?: number;
private isRequestingMedia = false;
private isUnmounted = false;
private shouldDiscardRecording = false;
@@ -72,6 +86,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
public componentWillUnmount(): void {
this.isUnmounted = true;
this.disposeRecording();
+ this.props.onRecordingStateChange(false);
}
public onRecordStartEndClick = async (): Promise<void> => {
@@ -96,6 +111,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
this.isRequestingMedia = true;
this.shouldDiscardRecording = false;
+ this.clearDurationCapTimer();
this.revokePreviewUrl();
this.recordedFile = undefined;
this.chunks = [];
@@ -108,6 +124,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
}
this.stream = stream;
+ this.bindStreamEndedHandlers(stream);
const mimeType = preferredVideoMimeType();
const mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
@@ -118,12 +135,13 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
};
mediaRecorder.onerror = (ev: Event): void => {
logger.error("Error recording video message:", ev);
- this.setState({ error: _t("composer|video_message_error") });
+ this.failRecording(_t("composer|video_message_error"));
};
mediaRecorder.onstop = this.onRecorderStop;
this.mediaRecorder = mediaRecorder;
mediaRecorder.start();
+ this.durationCapTimer = window.setTimeout(this.onDurationCap, MAX_VIDEO_RECORDING_MS);
this.setState({ error: undefined, isRecording: true, isSending: false, previewUrl: undefined });
this.props.onRecordingStateChange(true);
@@ -132,7 +150,9 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
this.stopStream();
if (!this.isUnmounted) {
this.setState({
- error: _t("composer|video_message_error"),
+ error: isPermissionDeniedError(e)
+ ? _t("composer|video_message_permission_denied")
+ : _t("composer|video_message_error"),
isRecording: false,
isSending: false,
previewUrl: undefined,
@@ -144,6 +164,21 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
}
};
+ private onDurationCap = (): void => {
+ this.durationCapTimer = undefined;
+
+ if (!this.mediaRecorder || this.mediaRecorder.state === "inactive") return;
+
+ this.stopRecording();
+ };
+
+ private onStreamTrackEnded = (): void => {
+ if (!this.mediaRecorder || this.mediaRecorder.state === "inactive") return;
+
+ logger.warn("Video message media stream ended before recording stopped");
+ this.failRecording(_t("composer|video_message_error"));
+ };
+
private stopRecording = (): void => {
if (!this.mediaRecorder) return;
@@ -160,6 +195,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
this.mediaRecorder = undefined;
+ this.clearDurationCapTimer();
this.stopStream();
if (this.isUnmounted) {
@@ -200,7 +236,9 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
this.props.replyToEvent,
);
this.clearRecording();
- this.props.onRecordingStateChange(false);
+ if (!this.isUnmounted) {
+ this.props.onRecordingStateChange(false);
+ }
} catch (e) {
logger.error("Error sending video message recording:", e);
if (!this.isUnmounted) {
@@ -219,13 +257,18 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
private clearRecording(): void {
this.chunks = [];
this.recordedFile = undefined;
+ this.clearDurationCapTimer();
+ this.stopStream();
this.revokePreviewUrl();
+ if (this.isUnmounted) return;
+
this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
}
private disposeRecording(): void {
this.chunks = [];
this.recordedFile = undefined;
+ this.clearDurationCapTimer();
this.stopStream();
this.revokePreviewUrl();
@@ -241,8 +284,32 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
}
}
+ private failRecording(error: string): void {
+ this.shouldDiscardRecording = true;
+ this.disposeRecording();
+
+ if (this.isUnmounted) return;
+
+ this.setState({ error, isRecording: false, isSending: false, previewUrl: undefined });
+ this.props.onRecordingStateChange(false);
+ }
+
+ private clearDurationCapTimer(): void {
+ if (this.durationCapTimer !== undefined) {
+ window.clearTimeout(this.durationCapTimer);
+ this.durationCapTimer = undefined;
+ }
+ }
+
+ private bindStreamEndedHandlers(stream: MediaStream): void {
+ stream.getTracks().forEach((track) => track.addEventListener("ended", this.onStreamTrackEnded));
+ }
+
private stopStream(): void {
- this.stream?.getTracks().forEach((track) => track.stop());
+ this.stream?.getTracks().forEach((track) => {
+ track.removeEventListener("ended", this.onStreamTrackEnded);
+ track.stop();
+ });
this.stream = undefined;
}
diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json
index e11fe90..83b7c21 100644
--- a/apps/web/src/i18n/strings/en_EN.json
+++ b/apps/web/src/i18n/strings/en_EN.json
@@ -648,6 +648,7 @@
"video_message_button": "Record video message",
"video_message_cancel": "Cancel video message",
"video_message_error": "Unable to record video message",
+ "video_message_permission_denied": "Camera permission denied",
"video_message_preview": "Video message preview",
"voice_message_button": "Voice Message"
},
+17
View File
@@ -0,0 +1,17 @@
# Element Web patches
Put ordered local patch files in this directory as `*.patch`.
- Files are applied in lexical order.
- Use zero-padded numeric prefixes when patch order matters, e.g. `0001-...patch`.
- Keep non-patch files out of the order by using a different extension; `README.md` is ignored by the Nix filter.
## Regenerating a patch
Create a clean checkout of upstream Element Web, make the change under `apps/web`, then run from the repository root:
```sh
git diff -- apps/web > package/element-web/patches/0001-description.patch
```
Use one patch per logical change so the sequence stays reproducible and reviewable.
@@ -0,0 +1,81 @@
{
bash,
cacert,
coreutils,
dockerTools,
git,
lib,
nix,
symlinkJoin,
}:
let
name = "gitea-runner-nix-image";
tag = "2026-06-07";
root = symlinkJoin {
name = "${name}-root";
paths = [
bash
cacert
coreutils
git
nix
];
};
in
dockerTools.buildLayeredImageWithNixDb {
inherit name tag;
contents = [ root ];
fakeRootCommands = ''
mkdir -p ./etc
cat > ./etc/passwd <<'EOF'
root:x:0:0:root:/root:/bin/bash
nobody:x:65534:65534:nobody:/var/empty:/bin/false
EOF
cat > ./etc/group <<'EOF'
root:x:0:
nixbld:x:30000:
nobody:x:65534:
EOF
for n in $(seq 1 10); do
echo "nixbld$n:x:$((30000 + n)):30000:Nix build user $n:/var/empty:/bin/false" >> ./etc/passwd
sed -i "s/^nixbld:x:30000:.*/&,nixbld$n/" ./etc/group
done
mkdir -p ./nix/var/nix/{gcroots,profiles,temproots,userpool,db}
mkdir -p ./nix/var/log/nix/drvs
chmod 1777 ./nix/var/nix/gcroots ./nix/var/nix/profiles
'';
extraCommands = ''
mkdir -p etc/nix root tmp
chmod 1777 tmp
cat > etc/nix/nix.conf <<'EOF'
accept-flake-config = true
experimental-features = nix-command flakes
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
trusted-users = root
sandbox = false
EOF
'';
config = {
Cmd = [ "${bash}/bin/bash" ];
Env = [
"HOME=/root"
"PATH=${lib.makeBinPath [ bash coreutils git nix ]}"
"SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
"USER=root"
];
Labels = {
"org.opencontainers.image.description" = "Nix-capable Gitea Actions job image";
"org.opencontainers.image.ref.name" = "${name}:${tag}";
"org.opencontainers.image.source" = "https://gitea.hectic-lab.com/hectic-lab/util.nix";
};
WorkingDir = "/workspace";
};
}
+1
View File
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
Commands: []*cli.Command{ Commands: []*cli.Command{
newUserCommand(), newUserCommand(),
newRepoSyncReleasesCommand(), newRepoSyncReleasesCommand(),
newHeatmapCommand(),
newRegenerateCommand(), newRegenerateCommand(),
newAuthCommand(), newAuthCommand(),
newSendMailCommand(), newSendMailCommand(),
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cmd
import (
"context"
"fmt"
"strings"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/urfave/cli/v3"
)
func newHeatmapCommand() *cli.Command {
return &cli.Command{
Name: "heatmap",
Usage: "Manage heatmap indexes",
Commands: []*cli.Command{
newHeatmapReindexCommand(),
},
}
}
func newHeatmapReindexCommand() *cli.Command {
return &cli.Command{
Name: "reindex",
Usage: "Reindex heatmap contributions from repository default branches",
Action: runHeatmapReindex,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "all",
Usage: "Reindex all repositories",
},
&cli.StringFlag{
Name: "repo",
Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
},
&cli.StringFlag{
Name: "owner",
Usage: "Owner name for --repo",
},
},
}
}
func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
if err := initDB(ctx); err != nil {
return err
}
if err := git.InitSimple(); err != nil {
return err
}
reindexAll := c.Bool("all")
repoName := c.String("repo")
ownerName := c.String("owner")
if reindexAll {
if repoName != "" || ownerName != "" {
return fmt.Errorf("--all cannot be combined with --repo or --owner")
}
return reindexAllHeatmapRepositories(ctx)
}
if repoName == "" {
return fmt.Errorf("either --all or --repo must be provided")
}
ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
if err != nil {
return err
}
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
if err != nil {
return err
}
return reindexHeatmapRepository(ctx, repo)
}
func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
if ownerName != "" {
if strings.Contains(repoName, "/") {
return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
}
return ownerName, repoName, nil
}
ownerName, repoName, ok := strings.Cut(repoName, "/")
if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
}
return ownerName, repoName, nil
}
func reindexAllHeatmapRepositories(ctx context.Context) error {
for page := 1; ; page++ {
repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
ListOptions: db.ListOptions{
PageSize: repo_model.RepositoryListDefaultPageSize,
Page: page,
},
Private: true,
})
if err != nil {
return fmt.Errorf("SearchRepositoryByName: %w", err)
}
if len(repos) == 0 {
break
}
log.Trace("Processing next %d repos of %d", len(repos), count)
for _, repo := range repos {
if err := reindexHeatmapRepository(ctx, repo); err != nil {
log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
continue
}
}
}
return nil
}
func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
}
return nil
}
@@ -0,0 +1,154 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cmd
import (
"fmt"
"strings"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/timeutil"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapAdminReindex(t *testing.T) {
repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
})
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
}
func TestHeatmapAdminRepoSelector(t *testing.T) {
owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
require.NoError(t, err)
assert.Equal(t, "user", owner)
assert.Equal(t, "repo", repo)
owner, repo, err = parseHeatmapRepoSelector("user", "repo")
require.NoError(t, err)
assert.Equal(t, "user", owner)
assert.Equal(t, "repo", repo)
_, _, err = parseHeatmapRepoSelector("", "repo")
assert.ErrorContains(t, err, "owner/name")
_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
assert.ErrorContains(t, err, "when --owner is set")
}
type heatmapAdminTestCommit struct {
Branch string
Mark string
Parent string
AuthorName string
AuthorEmail string
CommitterName string
CommitterEmail string
AuthorDate string
}
func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
t.Helper()
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
require.NoError(t, err)
marks := runHeatmapAdminFastImport(t, repo, commits)
repo.IsEmpty = false
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
return repo, marks
}
func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
t.Helper()
var stream strings.Builder
branchTips := make(map[string]string)
for i, commit := range commits {
mark := fmt.Sprintf(":%d", i+1)
branchRef := "refs/heads/" + commit.Branch
stream.WriteString("commit " + branchRef + "\n")
stream.WriteString("mark " + mark + "\n")
stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
message := "heatmap admin " + commit.Mark
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
if parentMark := branchTips[commit.Branch]; parentMark != "" {
stream.WriteString("from " + parentMark + "\n")
} else if commit.Parent != "" {
stream.WriteString("from " + commit.Parent + "\n")
}
content := commit.Mark + "\n"
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
branchTips[commit.Branch] = mark
}
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
WithDir(repo.RepoPath()).
WithStdinCopy(strings.NewReader(stream.String())).
Run(t.Context()))
commitSHAs := make(map[string]string, len(commits))
for _, commit := range commits {
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
AddDynamicArguments("refs/heads/" + commit.Branch).
WithDir(repo.RepoPath()).
RunStdString(t.Context())
require.NoError(t, err)
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
}
return commitSHAs
}
func heatmapAdminSignatureLine(kind, name, email, date string) string {
parsed, err := time.Parse(time.RFC3339, date)
if err != nil {
panic(err)
}
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
}
func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
t.Helper()
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
require.NoError(t, err)
return contributions
}
@@ -0,0 +1,121 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package activities
import (
"context"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/xorm/schemas"
)
// HeatmapContribution stores commit contributions counted for profile heatmaps.
type HeatmapContribution struct {
ID int64 `xorm:"pk autoincr"`
UserID int64 `xorm:"NOT NULL"`
RepoID int64 `xorm:"NOT NULL"`
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing.
type HeatmapContributionIdentity struct {
RepoID int64
CommitSHA string
UserID int64
}
func init() {
db.RegisterModel(new(HeatmapContribution))
}
// TableIndices implements xorm's TableIndices interface.
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
return []*schemas.Index{uniqueContribution, userAuthorRepo}
}
// UpsertHeatmapContribution creates or updates a commit contribution without duplicating counted rows.
func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContribution) error {
return db.WithTx(ctx, func(ctx context.Context) error {
e := db.GetEngine(ctx)
rows, err := e.Where("repo_id=? AND commit_sha=? AND user_id=?", contribution.RepoID, contribution.CommitSHA, contribution.UserID).
Cols("author_email", "author_unix").
Update(contribution)
if err != nil {
return err
}
if rows > 0 {
return nil
}
has, err := e.Exist(&HeatmapContribution{RepoID: contribution.RepoID, CommitSHA: contribution.CommitSHA, UserID: contribution.UserID})
if err != nil {
return err
}
if has {
return nil
}
_, err = e.Insert(contribution)
return err
})
}
// CountHeatmapContributions returns counted heatmap contribution rows for a user.
func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) {
return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution))
}
// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities.
func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error {
if len(identities) == 0 {
_, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution))
return err
}
current := make(map[HeatmapContributionIdentity]struct{}, len(identities))
for _, identity := range identities {
current[identity] = struct{}{}
}
contributions, err := FindHeatmapContributionsByRepo(ctx, repoID)
if err != nil {
return err
}
for _, contribution := range contributions {
identity := HeatmapContributionIdentity{
RepoID: contribution.RepoID,
CommitSHA: contribution.CommitSHA,
UserID: contribution.UserID,
}
if _, ok := current[identity]; ok {
continue
}
if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil {
return err
}
}
return nil
}
// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository.
func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) {
contributions := make([]*HeatmapContribution, 0)
return contributions, db.GetEngine(ctx).
Where("repo_id=?", repoID).
OrderBy("author_unix ASC, commit_sha ASC, user_id ASC").
Find(&contributions)
}
@@ -0,0 +1,81 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package activities_test
import (
"testing"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapContributionModel(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
assert.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
const (
userID = int64(2)
repoID = int64(1)
commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
authorUnix = timeutil.TimeStamp(1579089600)
)
contribution := &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: "user2@example.com",
AuthorUnix: authorUnix,
}
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), contribution))
count, err := activities_model.CountHeatmapContributions(t.Context(), userID)
require.NoError(t, err)
assert.EqualValues(t, 1, count)
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: "changed@example.com",
AuthorUnix: authorUnix + 900,
}))
count, err = activities_model.CountHeatmapContributions(t.Context(), userID)
require.NoError(t, err)
assert.EqualValues(t, 1, count)
stored := &activities_model.HeatmapContribution{UserID: userID, RepoID: repoID, CommitSHA: commitSHA}
has, err := db.GetEngine(t.Context()).Get(stored)
require.NoError(t, err)
require.True(t, has)
assert.Equal(t, "changed@example.com", stored.AuthorEmail)
assert.Equal(t, authorUnix+900, stored.AuthorUnix)
assert.NotZero(t, stored.CreatedUnix)
assert.NotZero(t, stored.UpdatedUnix)
err = db.Insert(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: "duplicate@example.com",
AuthorUnix: authorUnix,
})
assert.Error(t, err)
includePrivate, err := user_model.GetIncludePrivateContributions(t.Context(), userID)
require.NoError(t, err)
assert.False(t, includePrivate)
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), userID, true))
includePrivate, err = user_model.GetIncludePrivateContributions(t.Context(), userID)
require.NoError(t, err)
assert.True(t, includePrivate)
}
@@ -10,6 +10,7 @@ import (
"code.gitea.io/gitea/models/organization" "code.gitea.io/gitea/models/organization"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
) )
@@ -38,35 +39,45 @@ func getUserHeatmapData(ctx context.Context, user *user_model.User, team *organi
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone. // Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45. // The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
groupBy := "created_unix / 900 * 900" groupBy := "author_unix / 900 * 900"
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
switch { switch {
case setting.Database.Type.IsMySQL(): case setting.Database.Type.IsMySQL():
groupBy = "created_unix DIV 900 * 900" groupBy = "author_unix DIV 900 * 900"
case setting.Database.Type.IsMSSQL(): case setting.Database.Type.IsMSSQL():
groupByName = groupBy groupByName = groupBy
} }
cond, err := ActivityQueryCondition(ctx, GetFeedsOptions{ includePrivate, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
RequestedUser: user,
RequestedTeam: team,
Actor: doer,
IncludePrivate: true, // don't filter by private, as we already filter by repo access
IncludeDeleted: true,
// * Heatmaps for individual users only include actions that the user themself did.
// * For organizations actions by all users that were made in owned
// repositories are counted.
OnlyPerformedBy: !user.IsOrganization(),
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
return hdata, db.GetEngine(ctx). sess := db.GetEngine(ctx).
Select(groupBy+" AS timestamp, count(user_id) as contributions"). Select(groupBy+" AS timestamp, count(heatmap_contribution.user_id) as contributions").
Table("action"). Table("heatmap_contribution").
Where(cond). Join("INNER", "repository", "repository.id = heatmap_contribution.repo_id").
And("created_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap Join("INNER", "`user` AS repo_owner", "repo_owner.id = repository.owner_id").
Where("author_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
And("author_unix <= ?", timeutil.TimeStampNow())
if !includePrivate {
sess = sess.And("repository.is_private = ?", false).
And("repo_owner.visibility = ?", structs.VisibleTypePublic)
}
if user.IsOrganization() {
sess = sess.And("repository.owner_id = ?", user.ID)
if team != nil && !team.IncludesAllRepositories {
sess = sess.Join("INNER", "team_repo", "team_repo.repo_id = repository.id").
And("team_repo.org_id = ?", team.OrgID).
And("team_repo.team_id = ?", team.ID)
}
} else {
sess = sess.And("heatmap_contribution.user_id = ?", user.ID)
}
return hdata, sess.
GroupBy(groupByName). GroupBy(groupByName).
OrderBy("timestamp"). OrderBy("timestamp").
Find(&hdata) Find(&hdata)
@@ -4,19 +4,43 @@
package activities_test package activities_test
import ( import (
"crypto/sha1"
"fmt"
"testing" "testing"
"time" "time"
activities_model "code.gitea.io/gitea/models/activities" activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/organization"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestGetUserHeatmapDataByUser(t *testing.T) { func TestGetUserHeatmapDataByUser(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
// Mock time
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
insertHeatmapContribution(t, 2, 1, "user2-public-author-date", 1603009800)
insertHeatmapContribution(t, 2, 1, "user2-public-same-bucket", 1603009850)
insertHeatmapContribution(t, 2, 2, "user2-private-hidden", 1603010700)
insertHeatmapContribution(t, 2, 4, "user2-hidden-owner-hidden", 1603010700)
insertHeatmapContribution(t, 2, 1, "user2-future-hidden", timeutil.TimeStampNow()+900)
insertHeatmapContribution(t, 2, 999999, "user2-deleted-repo-hidden", 1603011600)
insertHeatmapContribution(t, 3, 1, "other-author-ignored", 1603010700)
setUserVisibility(t, 5, structs.VisibleTypePrivate)
testCases := []struct { testCases := []struct {
desc string desc string
userID int64 userID int64
@@ -25,73 +49,211 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
JSONResult string JSONResult string
}{ }{
{ {
"self looks at action in private repo", "self sees public author-date contributions only",
2, 2, 1, `[{"timestamp":1603227600,"contributions":1}]`, 2, 2, 2, `[{"timestamp":1603009800,"contributions":2}]`,
}, },
{ {
"admin looks at action in private repo", "admin sees public author-date contributions only",
2, 1, 1, `[{"timestamp":1603227600,"contributions":1}]`, 2, 1, 2, `[{"timestamp":1603009800,"contributions":2}]`,
}, },
{ {
"other user looks at action in private repo", "other user sees public author-date contributions only",
2, 3, 0, `[]`, 2, 3, 2, `[{"timestamp":1603009800,"contributions":2}]`,
}, },
{ {
"nobody looks at action in private repo", "anonymous sees public author-date contributions only",
2, 0, 0, `[]`, 2, 0, 2, `[{"timestamp":1603009800,"contributions":2}]`,
}, },
{ {
"collaborator looks at action in private repo", "different author is not counted for target user",
16, 15, 1, `[{"timestamp":1603267200,"contributions":1}]`,
},
{
"no action action not performed by target user",
3, 3, 0, `[]`, 3, 3, 0, `[]`,
}, },
{
"multiple actions performed with two grouped together",
10, 10, 3, `[{"timestamp":1603009800,"contributions":1},{"timestamp":1603010700,"contributions":2}]`,
},
} }
// Prepare
assert.NoError(t, unittest.PrepareTestDatabase())
// Mock time for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
var doer *user_model.User
if tc.doerID != 0 {
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
}
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
require.NoError(t, err)
assert.Equal(t, tc.CountResult, countHeatmapContributions(heatmap), "testcase '%s'", tc.desc)
// Test JSON rendering
jsonData, err := json.Marshal(heatmap)
require.NoError(t, err)
assert.JSONEq(t, tc.JSONResult, string(jsonData))
})
}
}
func TestGetUserHeatmapDataByUserUsesCurrentRepoVisibility(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)) timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset() defer timeutil.MockUnset()
for _, tc := range testCases { insertHeatmapContribution(t, 2, 1, "user2-visibility-flip", 1603009800)
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
var doer *user_model.User target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
if tc.doerID != 0 { heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID}) require.NoError(t, err)
} assertHeatmapJSON(t, heatmap, `[{
"timestamp": 1603009800,
"contributions": 1
}]`)
// get the action for comparison setRepoPrivate(t, 1, true)
actions, count, err := activities_model.GetFeeds(t.Context(), activities_model.GetFeedsOptions{ heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
RequestedUser: user, require.NoError(t, err)
Actor: doer, assert.Empty(t, heatmap)
IncludePrivate: true,
OnlyPerformedBy: true,
IncludeDeleted: true,
})
assert.NoError(t, err)
// Get the heatmap and compare require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, true))
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer) heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
var contributions int require.NoError(t, err)
for _, hm := range heatmap { assertHeatmapJSON(t, heatmap, `[{
contributions += int(hm.Contributions) "timestamp": 1603009800,
} "contributions": 1
assert.NoError(t, err) }]`)
assert.Len(t, actions, contributions, "invalid action count: did the test data became too old?") }
assert.Equal(t, count, int64(contributions))
assert.Equal(t, tc.CountResult, contributions, "testcase '%s'", tc.desc)
// Test JSON rendering func TestGetUserHeatmapDataByOrgTeam(t *testing.T) {
jsonData, err := json.Marshal(heatmap) require.NoError(t, unittest.PrepareTestDatabase())
assert.NoError(t, err) require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
assert.JSONEq(t, tc.JSONResult, string(jsonData)) require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 3, false))
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
insertHeatmapContribution(t, 2, 32, "org-team-public", 1603009800)
insertHeatmapContribution(t, 2, 3, "org-team-private-hidden", 1603010700)
insertHeatmapContribution(t, 2, 1, "non-org-repo-ignored", 1603011600)
org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3})
team := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: 7})
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
heatmap, err := activities_model.GetUserHeatmapDataByOrgTeam(t.Context(), org, team, doer)
require.NoError(t, err)
assert.Equal(t, 1, countHeatmapContributions(heatmap))
jsonData, err := json.Marshal(heatmap)
require.NoError(t, err)
assert.JSONEq(t, `[{"timestamp":1603009800,"contributions":1}]`, string(jsonData))
}
func TestUserHeatmapPrivateContributionsOptIn(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
insertHeatmapContribution(t, 16, 21, "user16-public", 1603009800)
insertHeatmapContribution(t, 16, 22, "user16-private-one", 1603009850)
insertHeatmapContribution(t, 16, 22, "user16-private-two", 1603010700)
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 16})
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
authenticated := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3})
collaborator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15})
for _, doer := range []*user_model.User{nil, target, admin, authenticated, collaborator} {
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, doer)
require.NoError(t, err)
assert.Equal(t, 1, countHeatmapContributions(heatmap), "private contributions should be hidden by default from %s", heatmapDoerName(doer))
}
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, true))
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assertHeatmapJSON(t, heatmap, `[{"timestamp":1603009800,"contributions":2},{"timestamp":1603010700,"contributions":1}]`)
target.KeepActivityPrivate = true
_, err = db.GetEngine(t.Context()).ID(target.ID).Cols("keep_activity_private").Update(target)
require.NoError(t, err)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
require.NoError(t, err)
assert.Empty(t, heatmap)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, authenticated)
require.NoError(t, err)
assert.Empty(t, heatmap)
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, target)
require.NoError(t, err)
assert.Equal(t, 3, countHeatmapContributions(heatmap))
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, admin)
require.NoError(t, err)
assert.Equal(t, 3, countHeatmapContributions(heatmap))
}
func insertHeatmapContribution(t *testing.T, userID, repoID int64, commitSHA string, authorUnix timeutil.TimeStamp) {
t.Helper()
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: fmt.Sprintf("%040x", sha1.Sum([]byte(commitSHA))),
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
AuthorUnix: authorUnix,
}))
}
func setRepoPrivate(t *testing.T, repoID int64, isPrivate bool) {
t.Helper()
repo := &repo_model.Repository{ID: repoID, IsPrivate: isPrivate}
_, err := db.GetEngine(t.Context()).ID(repoID).Cols("is_private").Update(repo)
require.NoError(t, err)
}
func setUserVisibility(t *testing.T, userID int64, visibility structs.VisibleType) {
t.Helper()
user := &user_model.User{ID: userID, Visibility: visibility}
_, err := db.GetEngine(t.Context()).ID(userID).Cols("visibility").Update(user)
require.NoError(t, err)
}
func assertHeatmapJSON(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected string) {
t.Helper()
jsonData, err := json.Marshal(heatmap)
require.NoError(t, err)
assert.JSONEq(t, expected, string(jsonData))
var decoded []map[string]any
require.NoError(t, json.Unmarshal(jsonData, &decoded))
for _, entry := range decoded {
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, mapKeys(entry))
} }
} }
func mapKeys[K comparable, V any](m map[K]V) []K {
keys := make([]K, 0, len(m))
for key := range m {
keys = append(keys, key)
}
return keys
}
func countHeatmapContributions(heatmap []*activities_model.UserHeatmapData) int {
var contributions int
for _, hm := range heatmap {
contributions += int(hm.Contributions)
}
return contributions
}
func heatmapDoerName(doer *user_model.User) string {
if doer == nil {
return "anonymous"
}
return doer.Name
}
@@ -405,6 +405,7 @@ func prepareMigrationTasks() []*migration {
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob), newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge), newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook), newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
newMigration(331, "Create heatmap contribution table", v1_26.CreateHeatmapContributionTable),
} }
return preparedMigrations return preparedMigrations
} }
@@ -0,0 +1,38 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_26
import (
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/xorm"
"xorm.io/xorm/schemas"
)
type HeatmapContribution struct { //revive:disable-line:exported
ID int64 `xorm:"pk autoincr"`
UserID int64 `xorm:"NOT NULL"`
RepoID int64 `xorm:"NOT NULL"`
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
// TableIndices implements xorm's TableIndices interface.
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
return []*schemas.Index{uniqueContribution, userAuthorRepo}
}
func CreateHeatmapContributionTable(x *xorm.Engine) error {
_, err := x.SyncWithOptions(xorm.SyncOptions{IgnoreDropIndices: true}, new(HeatmapContribution))
return err
}
@@ -7,6 +7,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"strconv"
"strings" "strings"
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
@@ -19,6 +20,8 @@ import (
"xorm.io/xorm/convert" "xorm.io/xorm/convert"
) )
const SettingsKeyIncludePrivateContributions = "include_private_contributions"
// Setting is a key value store of user settings // Setting is a key value store of user settings
type Setting struct { type Setting struct {
ID int64 `xorm:"pk autoincr"` ID int64 `xorm:"pk autoincr"`
@@ -254,3 +257,17 @@ func SetUserSettingJSON[T any](ctx context.Context, userID int64, key string, va
} }
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs)) return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
} }
// GetIncludePrivateContributions returns whether a user opted in to private heatmap contributions.
func GetIncludePrivateContributions(ctx context.Context, userID int64) (bool, error) {
value, err := GetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, "false")
if err != nil {
return false, err
}
return strconv.ParseBool(value)
}
// SetIncludePrivateContributions stores whether a user opted in to private heatmap contributions.
func SetIncludePrivateContributions(ctx context.Context, userID int64, include bool) error {
return SetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, strconv.FormatBool(include))
}
+6 -4
View File
@@ -80,8 +80,9 @@ type UserSettings struct {
Theme string `json:"theme"` Theme string `json:"theme"`
DiffViewStyle string `json:"diff_view_style"` DiffViewStyle string `json:"diff_view_style"`
// Privacy // Privacy
HideEmail bool `json:"hide_email"` HideEmail bool `json:"hide_email"`
HideActivity bool `json:"hide_activity"` HideActivity bool `json:"hide_activity"`
IncludePrivateContributions bool `json:"include_private_contributions"`
} }
// UserSettingsOptions represents options to change user settings // UserSettingsOptions represents options to change user settings
@@ -95,8 +96,9 @@ type UserSettingsOptions struct {
Theme *string `json:"theme"` Theme *string `json:"theme"`
DiffViewStyle *string `json:"diff_view_style"` DiffViewStyle *string `json:"diff_view_style"`
// Privacy // Privacy
HideEmail *bool `json:"hide_email"` HideEmail *bool `json:"hide_email"`
HideActivity *bool `json:"hide_activity"` HideActivity *bool `json:"hide_activity"`
IncludePrivateContributions *bool `json:"include_private_contributions"`
} }
// RenameUserOption options when renaming a user // RenameUserOption options when renaming a user
@@ -708,6 +708,8 @@
"settings.privacy": "Privacy", "settings.privacy": "Privacy",
"settings.keep_activity_private": "Hide Activity from profile page", "settings.keep_activity_private": "Hide Activity from profile page",
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins", "settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
"settings.include_private_contributions": "Show private contributions on profile heatmap",
"settings.include_private_contributions_popup": "Publicly reveals only contribution dates and counts from non-public repositories, never repository or commit details.",
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address", "settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
"settings.federated_avatar_lookup": "Federated Avatar Lookup", "settings.federated_avatar_lookup": "Federated Avatar Lookup",
"settings.enable_custom_avatar": "Use Custom Avatar", "settings.enable_custom_avatar": "Use Custom Avatar",
@@ -24,7 +24,12 @@ func GetUserSettings(ctx *context.APIContext) {
// responses: // responses:
// "200": // "200":
// "$ref": "#/responses/UserSettings" // "$ref": "#/responses/UserSettings"
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer)) settings, err := convert.User2UserSettings(ctx, ctx.Doer)
if err != nil {
ctx.APIErrorInternal(err)
return
}
ctx.JSON(http.StatusOK, settings)
} }
// UpdateUserSettings returns user settings // UpdateUserSettings returns user settings
@@ -46,20 +51,26 @@ func UpdateUserSettings(ctx *context.APIContext) {
form := web.GetForm(ctx).(*api.UserSettingsOptions) form := web.GetForm(ctx).(*api.UserSettingsOptions)
opts := &user_service.UpdateOptions{ opts := &user_service.UpdateOptions{
FullName: optional.FromPtr(form.FullName), FullName: optional.FromPtr(form.FullName),
Description: optional.FromPtr(form.Description), Description: optional.FromPtr(form.Description),
Website: optional.FromPtr(form.Website), Website: optional.FromPtr(form.Website),
Location: optional.FromPtr(form.Location), Location: optional.FromPtr(form.Location),
Language: optional.FromPtr(form.Language), Language: optional.FromPtr(form.Language),
Theme: optional.FromPtr(form.Theme), Theme: optional.FromPtr(form.Theme),
DiffViewStyle: optional.FromPtr(form.DiffViewStyle), DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
KeepEmailPrivate: optional.FromPtr(form.HideEmail), KeepEmailPrivate: optional.FromPtr(form.HideEmail),
KeepActivityPrivate: optional.FromPtr(form.HideActivity), KeepActivityPrivate: optional.FromPtr(form.HideActivity),
IncludePrivateContributions: optional.FromPtr(form.IncludePrivateContributions),
} }
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil { if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
ctx.APIErrorInternal(err) ctx.APIErrorInternal(err)
return return
} }
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer)) settings, err := convert.User2UserSettings(ctx, ctx.Doer)
if err != nil {
ctx.APIErrorInternal(err)
return
}
ctx.JSON(http.StatusOK, settings)
} }
@@ -48,16 +48,32 @@ func Profile(ctx *context.Context) {
ctx.Data["PageIsSettingsProfile"] = true ctx.Data["PageIsSettingsProfile"] = true
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice() ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx) ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
if !loadProfilePrivateContributionSetting(ctx) {
return
}
ctx.HTML(http.StatusOK, tplSettingsProfile) ctx.HTML(http.StatusOK, tplSettingsProfile)
} }
func loadProfilePrivateContributionSetting(ctx *context.Context) bool {
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, ctx.Doer.ID)
if err != nil {
ctx.ServerError("GetIncludePrivateContributions", err)
return false
}
ctx.Data["IncludePrivateContributions"] = includePrivateContributions
return true
}
// ProfilePost response for change user's profile // ProfilePost response for change user's profile
func ProfilePost(ctx *context.Context) { func ProfilePost(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("settings_title") ctx.Data["Title"] = ctx.Tr("settings_title")
ctx.Data["PageIsSettingsProfile"] = true ctx.Data["PageIsSettingsProfile"] = true
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice() ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx) ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
if !loadProfilePrivateContributionSetting(ctx) {
return
}
if ctx.HasError() { if ctx.HasError() {
ctx.HTML(http.StatusOK, tplSettingsProfile) ctx.HTML(http.StatusOK, tplSettingsProfile)
@@ -94,12 +110,13 @@ func ProfilePost(ctx *context.Context) {
} }
opts := &user_service.UpdateOptions{ opts := &user_service.UpdateOptions{
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate), KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
Description: optional.Some(form.Description), Description: optional.Some(form.Description),
Website: optional.Some(form.Website), Website: optional.Some(form.Website),
Location: optional.Some(form.Location), Location: optional.Some(form.Location),
Visibility: optional.Some(form.Visibility), Visibility: optional.Some(form.Visibility),
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate), KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
IncludePrivateContributions: optional.Some(form.IncludePrivateContributions),
} }
if form.FullName != "" { if form.FullName != "" {
+16 -11
View File
@@ -86,18 +86,23 @@ func toUser(ctx context.Context, user *user_model.User, signed, authed bool) *ap
} }
// User2UserSettings return UserSettings based on a user // User2UserSettings return UserSettings based on a user
func User2UserSettings(user *user_model.User) api.UserSettings { func User2UserSettings(ctx context.Context, user *user_model.User) (api.UserSettings, error) {
return api.UserSettings{ includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
FullName: user.FullName, if err != nil {
Website: user.Website, return api.UserSettings{}, err
Location: user.Location,
Language: user.Language,
Description: user.Description,
Theme: user.Theme,
HideEmail: user.KeepEmailPrivate,
HideActivity: user.KeepActivityPrivate,
DiffViewStyle: user.DiffViewStyle,
} }
return api.UserSettings{
FullName: user.FullName,
Website: user.Website,
Location: user.Location,
Language: user.Language,
Description: user.Description,
Theme: user.Theme,
HideEmail: user.KeepEmailPrivate,
HideActivity: user.KeepActivityPrivate,
IncludePrivateContributions: includePrivateContributions,
DiffViewStyle: user.DiffViewStyle,
}, nil
} }
// ToUserAndPermission return User and its collaboration permission for a repository // ToUserAndPermission return User and its collaboration permission for a repository
@@ -208,14 +208,15 @@ func (f *IntrospectTokenForm) Validate(req *http.Request, errs binding.Errors) b
// UpdateProfileForm form for updating profile // UpdateProfileForm form for updating profile
type UpdateProfileForm struct { type UpdateProfileForm struct {
Name string `binding:"Username;MaxSize(40)"` Name string `binding:"Username;MaxSize(40)"`
FullName string `binding:"MaxSize(100)"` FullName string `binding:"MaxSize(100)"`
KeepEmailPrivate bool KeepEmailPrivate bool
Website string `binding:"ValidSiteUrl;MaxSize(255)"` Website string `binding:"ValidSiteUrl;MaxSize(255)"`
Location string `binding:"MaxSize(50)"` Location string `binding:"MaxSize(50)"`
Description string `binding:"MaxSize(255)"` Description string `binding:"MaxSize(255)"`
Visibility structs.VisibleType Visibility structs.VisibleType
KeepActivityPrivate bool KeepActivityPrivate bool
IncludePrivateContributions bool
} }
// Validate validates the fields // Validate validates the fields
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
defer gitRepo.Close() defer gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
indexHeatmap := false
if len(results) > 0 { if len(results) > 0 {
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok { if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err) log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
if result.RefName.IsPull() { if result.RefName.IsPull() {
continue continue
} }
if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
indexHeatmap = true
}
// Create reference // Create reference
if result.OldCommitID == "" { if result.OldCommitID == "" {
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
NewCommitID: newCommitID, NewCommitID: newCommitID,
}, theCommits) }, theCommits)
} }
if indexHeatmap {
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
}
}
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
isEmpty, err := gitRepo.IsEmpty() isEmpty, err := gitRepo.IsEmpty()
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
} }
notify_service.ChangeDefaultBranch(ctx, repo) notify_service.ChangeDefaultBranch(ctx, repo)
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
}
return nil return nil
} }
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
} }
} }
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
}
return repo, nil return repo, nil
} }
@@ -0,0 +1,109 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"context"
"strings"
activities_model "code.gitea.io/gitea/models/activities"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/timeutil"
)
const heatmapIndexCommitsPageSize = 100
// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch.
func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error {
if repo == nil || repo.ID == 0 {
return nil
}
if repo.DefaultBranch == "" || repo.IsEmpty {
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
}
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
if err != nil {
return err
}
defer closer.Close()
head, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
if git.IsErrNotExist(err) {
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
}
if err != nil {
return err
}
currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0)
now := timeutil.TimeStampNow()
for page := 1; ; page++ {
commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "")
if err != nil {
return err
}
if len(commits) == 0 {
break
}
for _, commit := range commits {
commitSHA := commit.ID.String()
if commit.Author == nil || commit.Author.Email == "" {
continue
}
authorUnix := timeutil.TimeStamp(commit.Author.When.Unix())
if authorUnix > now {
continue
}
userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email)
if err != nil {
return err
}
if !ok {
continue
}
if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repo.ID,
CommitSHA: commitSHA,
AuthorEmail: strings.ToLower(commit.Author.Email),
AuthorUnix: authorUnix,
}); err != nil {
return err
}
currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{
RepoID: repo.ID,
CommitSHA: commitSHA,
UserID: userID,
})
}
if len(commits) < heatmapIndexCommitsPageSize {
break
}
}
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities)
}
func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) {
address, err := user_model.GetEmailAddressByEmail(ctx, email)
if user_model.IsErrEmailAddressNotExist(err) {
return 0, false, nil
}
if err != nil {
return 0, false, err
}
if !address.IsActivated {
return 0, false, nil
}
return address.UID, true, nil
}
@@ -0,0 +1,218 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"fmt"
"strings"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{
{Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
{Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
{Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, int64(2), contributions[0].UserID)
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail)
assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix)
emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com")
require.NoError(t, err)
emailAddress.UID = 1
require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress))
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
contributions = loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, int64(1), contributions[0].UserID)
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
}
func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{
{Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
{Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
{Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 2)
assert.Equal(t, int64(2), contributions[0].UserID)
assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA)
assert.Equal(t, int64(1), contributions[1].UserID)
assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA)
for _, contribution := range contributions {
assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA)
}
deleteMainRef(t, repo)
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
}
func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
})
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
{
RefFullName: git.RefNameFromBranch("main"),
OldCommitID: commits["initial"],
NewCommitID: newCommits["pushed"],
PusherID: 1,
RepoUserName: repo.OwnerName,
RepoName: repo.Name,
},
}))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 2)
assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
})
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
{
RefFullName: git.RefNameFromBranch("feature"),
OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(),
NewCommitID: featureCommits["feature-pushed"],
PusherID: 1,
RepoUserName: repo.OwnerName,
RepoName: repo.Name,
},
}))
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
}
type heatmapIndexTestCommit struct {
Branch string
Mark string
Parent string
AuthorName string
AuthorEmail string
CommitterName string
CommitterEmail string
AuthorDate string
}
func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) {
t.Helper()
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
require.NoError(t, err)
marks := runFastImport(t, repo, commits)
repo.IsEmpty = false
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
return repo, marks
}
func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string {
t.Helper()
var stream strings.Builder
branchTips := make(map[string]string)
for i, commit := range commits {
mark := fmt.Sprintf(":%d", i+1)
branchRef := "refs/heads/" + commit.Branch
stream.WriteString("commit " + branchRef + "\n")
stream.WriteString("mark " + mark + "\n")
stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
message := "heatmap " + commit.Mark
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
if parentMark := branchTips[commit.Branch]; parentMark != "" {
stream.WriteString("from " + parentMark + "\n")
} else if commit.Parent != "" {
stream.WriteString("from " + commit.Parent + "\n")
}
content := commit.Mark + "\n"
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
branchTips[commit.Branch] = mark
}
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
WithDir(repo.RepoPath()).
WithStdinCopy(strings.NewReader(stream.String())).
Run(t.Context()))
commitSHAs := make(map[string]string, len(commits))
for _, commit := range commits {
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
AddOptionFormat("--grep=%s", "heatmap "+commit.Mark).
AddDynamicArguments("refs/heads/" + commit.Branch).
WithDir(repo.RepoPath()).
RunStdString(t.Context())
require.NoError(t, err)
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
}
return commitSHAs
}
func signatureLine(kind, name, email, date string) string {
parsed, err := time.Parse(time.RFC3339, date)
if err != nil {
panic(err)
}
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
}
func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
t.Helper()
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
require.NoError(t, err)
return contributions
}
func deleteMainRef(t *testing.T, repo *repo_model.Repository) {
t.Helper()
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
}
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
} }
} }
return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) { repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
if opts.Mirror { if opts.Mirror {
remoteAddress, err := util.SanitizeURL(opts.CloneAddr) remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
if err != nil { if err != nil {
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
} }
return repo, nil return repo, nil
}) })
if err != nil {
return nil, err
}
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
}
return repo, nil
} }
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
} }
} }
indexHeatmap := false
if !opts.IsDelRef() { if !opts.IsDelRef() {
branch := opts.RefFullName.BranchName() branch := opts.RefFullName.BranchName()
@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil { if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
log.Error("DelRepoDivergenceFromCache: %v", err) log.Error("DelRepoDivergenceFromCache: %v", err)
} }
indexHeatmap = true
} else { } else {
if err := DelDivergenceFromCache(repo.ID, branch); err != nil { if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
log.Error("DelDivergenceFromCache: %v", err) log.Error("DelDivergenceFromCache: %v", err)
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
pushDeleteBranch(ctx, repo, pusher, opts) pushDeleteBranch(ctx, repo, pusher, opts)
} }
if indexHeatmap {
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
}
}
// Even if user delete a branch on a repository which he didn't watch, he will be watch that. // Even if user delete a branch on a repository which he didn't watch, he will be watch that.
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil { if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
+19 -1
View File
@@ -8,6 +8,7 @@ import (
"fmt" "fmt"
auth_model "code.gitea.io/gitea/models/auth" auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
password_module "code.gitea.io/gitea/modules/auth/password" password_module "code.gitea.io/gitea/modules/auth/password"
"code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/optional"
@@ -47,6 +48,7 @@ type UpdateOptions struct {
IsRestricted optional.Option[bool] IsRestricted optional.Option[bool]
Visibility optional.Option[structs.VisibleType] Visibility optional.Option[structs.VisibleType]
KeepActivityPrivate optional.Option[bool] KeepActivityPrivate optional.Option[bool]
IncludePrivateContributions optional.Option[bool]
Language optional.Option[string] Language optional.Option[string]
Theme optional.Option[string] Theme optional.Option[string]
DiffViewStyle optional.Option[string] DiffViewStyle optional.Option[string]
@@ -182,7 +184,23 @@ func UpdateUser(ctx context.Context, u *user_model.User, opts *UpdateOptions) er
cols = append(cols, "last_login_unix") cols = append(cols, "last_login_unix")
} }
return user_model.UpdateUserCols(ctx, u, cols...) if len(cols) > 0 || opts.IncludePrivateContributions.Has() {
return db.WithTx(ctx, func(ctx context.Context) error {
if len(cols) > 0 {
if err := user_model.UpdateUserCols(ctx, u, cols...); err != nil {
return err
}
}
if opts.IncludePrivateContributions.Has() {
return user_model.SetIncludePrivateContributions(ctx, u.ID, opts.IncludePrivateContributions.Value())
}
return nil
})
}
return nil
} }
type UpdateAuthOptions struct { type UpdateAuthOptions struct {
@@ -44,6 +44,7 @@ func TestUpdateUser(t *testing.T) {
IsAdmin: UpdateOptionFieldFromValue(true), IsAdmin: UpdateOptionFieldFromValue(true),
Visibility: optional.Some(structs.VisibleTypePrivate), Visibility: optional.Some(structs.VisibleTypePrivate),
KeepActivityPrivate: optional.Some(true), KeepActivityPrivate: optional.Some(true),
IncludePrivateContributions: optional.Some(true),
Language: optional.Some("lang"), Language: optional.Some("lang"),
Theme: optional.Some("theme"), Theme: optional.Some("theme"),
DiffViewStyle: optional.Some("split"), DiffViewStyle: optional.Some("split"),
@@ -66,6 +67,9 @@ func TestUpdateUser(t *testing.T) {
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin) assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
assert.Equal(t, opts.Visibility.Value(), user.Visibility) assert.Equal(t, opts.Visibility.Value(), user.Visibility)
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate) assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
assert.NoError(t, err)
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
assert.Equal(t, opts.Language.Value(), user.Language) assert.Equal(t, opts.Language.Value(), user.Language)
assert.Equal(t, opts.Theme.Value(), user.Theme) assert.Equal(t, opts.Theme.Value(), user.Theme)
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle) assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
@@ -86,6 +90,9 @@ func TestUpdateUser(t *testing.T) {
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin) assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
assert.Equal(t, opts.Visibility.Value(), user.Visibility) assert.Equal(t, opts.Visibility.Value(), user.Visibility)
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate) assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
includePrivateContributions, err = user_model.GetIncludePrivateContributions(t.Context(), user.ID)
assert.NoError(t, err)
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
assert.Equal(t, opts.Language.Value(), user.Language) assert.Equal(t, opts.Language.Value(), user.Language)
assert.Equal(t, opts.Theme.Value(), user.Theme) assert.Equal(t, opts.Theme.Value(), user.Theme)
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle) assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
+8
View File
@@ -29655,6 +29655,10 @@
"type": "boolean", "type": "boolean",
"x-go-name": "HideEmail" "x-go-name": "HideEmail"
}, },
"include_private_contributions": {
"type": "boolean",
"x-go-name": "IncludePrivateContributions"
},
"language": { "language": {
"type": "string", "type": "string",
"x-go-name": "Language" "x-go-name": "Language"
@@ -29699,6 +29703,10 @@
"type": "boolean", "type": "boolean",
"x-go-name": "HideEmail" "x-go-name": "HideEmail"
}, },
"include_private_contributions": {
"type": "boolean",
"x-go-name": "IncludePrivateContributions"
},
"language": { "language": {
"type": "string", "type": "string",
"x-go-name": "Language" "x-go-name": "Language"
@@ -88,6 +88,13 @@
</div> </div>
</div> </div>
<div class="field">
<div class="ui checkbox" id="include-private-contributions">
<label data-tooltip-content="{{ctx.Locale.Tr "settings.include_private_contributions_popup"}}"><strong>{{ctx.Locale.Tr "settings.include_private_contributions"}}</strong></label>
<input name="include_private_contributions" type="checkbox" {{if .IncludePrivateContributions}}checked{{end}}>
</div>
</div>
<div class="divider"></div> <div class="divider"></div>
<div class="field"> <div class="field">
@@ -4,17 +4,19 @@
package integration package integration
import ( import (
"fmt"
"net/http" "net/http"
"testing" "testing"
"time" "time"
activities_model "code.gitea.io/gitea/models/activities" activities_model "code.gitea.io/gitea/models/activities"
auth_model "code.gitea.io/gitea/models/auth" auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/tests" "code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestUserHeatmap(t *testing.T) { func TestUserHeatmap(t *testing.T) {
@@ -23,17 +25,23 @@ func TestUserHeatmap(t *testing.T) {
normalUsername := "user2" normalUsername := "user2"
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser) token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
fakeNow := time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local) fakeNow := time.Date(2011, 10, 21, 0, 0, 0, 0, time.Local)
timeutil.MockSet(fakeNow) timeutil.MockSet(fakeNow)
defer timeutil.MockUnset() defer timeutil.MockUnset()
req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)). require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-author-date", 1319068800)
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-same-bucket", 1319068850)
testHeatmapSeedContribution(t, 2, 2, "api-user2-private-hidden", 1319070600)
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", normalUsername).
AddTokenAuth(token) AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK) resp := MakeRequest(t, req, http.StatusOK)
var heatmap []*activities_model.UserHeatmapData var heatmap []*activities_model.UserHeatmapData
DecodeJSON(t, resp, &heatmap) DecodeJSON(t, resp, &heatmap)
var dummyheatmap []*activities_model.UserHeatmapData
dummyheatmap = append(dummyheatmap, &activities_model.UserHeatmapData{Timestamp: 1603227600, Contributions: 1})
assert.Equal(t, dummyheatmap, heatmap) assert.Equal(t, []*activities_model.UserHeatmapData{
{Timestamp: 1319068800, Contributions: 2},
}, heatmap)
} }
@@ -4,22 +4,40 @@
package integration package integration
import ( import (
"crypto/sha1"
"fmt"
"net/http" "net/http"
"net/http/httptest"
"testing" "testing"
"time" "time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/tests" "code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
type testHeatmapWebResponse struct {
HeatmapData [][2]int64 `json:"heatmapData"`
TotalContributions int64 `json:"totalContributions"`
}
func TestHeatmapEndpoints(t *testing.T) { func TestHeatmapEndpoints(t *testing.T) {
defer tests.PrepareTestEnv(t)() defer tests.PrepareTestEnv(t)()
// Mock time so fixture actions fall within the heatmap's time window // Mock time so fixture actions fall within the heatmap's time window
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)) timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset() defer timeutil.MockUnset()
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-one", 1603009800)
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-two", 1603009850)
testHeatmapSeedContribution(t, 2, 2, "web-user2-private-hidden", 1603010700)
session := loginUser(t, "user2") session := loginUser(t, "user2")
@@ -28,11 +46,15 @@ func TestHeatmapEndpoints(t *testing.T) {
req := NewRequest(t, "GET", "/user2/-/heatmap") req := NewRequest(t, "GET", "/user2/-/heatmap")
resp := session.MakeRequest(t, req, http.StatusOK) resp := session.MakeRequest(t, req, http.StatusOK)
var result map[string]any webHeatmap := testHeatmapDecodeWebResponse(t, resp)
DecodeJSON(t, resp, &result)
assert.Contains(t, result, "heatmapData") req = NewRequest(t, "GET", "/api/v1/users/user2/heatmap")
assert.Contains(t, result, "totalContributions") resp = session.MakeRequest(t, req, http.StatusOK)
assert.Positive(t, result["totalContributions"]) var apiHeatmap []*activities_model.UserHeatmapData
DecodeJSON(t, resp, &apiHeatmap)
assert.Equal(t, testHeatmapSumAPIContributions(apiHeatmap), webHeatmap.TotalContributions)
assert.Equal(t, int64(2), webHeatmap.TotalContributions)
}) })
t.Run("OrgDashboard", func(t *testing.T) { t.Run("OrgDashboard", func(t *testing.T) {
@@ -57,3 +79,71 @@ func TestHeatmapEndpoints(t *testing.T) {
assert.Contains(t, result, "totalContributions") assert.Contains(t, result, "totalContributions")
}) })
} }
func testHeatmapSeedContribution(t *testing.T, userID, repoID int64, label string, authorUnix timeutil.TimeStamp) string {
t.Helper()
commitSHA := fmt.Sprintf("%040x", sha1.Sum([]byte(label)))
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
UserID: userID,
RepoID: repoID,
CommitSHA: commitSHA,
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
AuthorUnix: authorUnix,
}))
return commitSHA
}
func testHeatmapDecodeWebResponse(t *testing.T, resp *httptest.ResponseRecorder) testHeatmapWebResponse {
t.Helper()
var result testHeatmapWebResponse
DecodeJSON(t, resp, &result)
return result
}
func testHeatmapSumAPIContributions(heatmap []*activities_model.UserHeatmapData) int64 {
var total int64
for _, item := range heatmap {
total += item.Contributions
}
return total
}
func testHeatmapAssertAggregateOnlyAPIResponse(t *testing.T, body []byte) {
t.Helper()
var decoded []map[string]any
require.NoError(t, json.Unmarshal(body, &decoded))
for _, entry := range decoded {
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, testHeatmapMapKeys(entry))
}
}
func testHeatmapAssertAggregateOnlyWebResponse(t *testing.T, body []byte) {
t.Helper()
var decoded map[string]any
require.NoError(t, json.Unmarshal(body, &decoded))
assert.ElementsMatch(t, []string{"heatmapData", "totalContributions"}, testHeatmapMapKeys(decoded))
}
func testHeatmapAssertNoPrivateMetadata(t *testing.T, body []byte, forbidden ...string) {
t.Helper()
response := string(body)
for _, value := range forbidden {
assert.NotContains(t, response, value)
}
for _, value := range []string{"repo_id", "repository_id", "repoID", "commit_sha", "commit_message", "branch", "url", "action_id"} {
assert.NotContains(t, response, value)
}
}
func testHeatmapMapKeys[K comparable, V any](m map[K]V) []K {
keys := make([]K, 0, len(m))
for key := range m {
keys = append(keys, key)
}
return keys
}

Some files were not shown because too many files have changed in this diff Show More