forked from hinterland/hearth
Compare commits
67 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 557b6e9ef0 | |||
| e49f497045 | |||
| feb1a48db1 | |||
| 30732080b7 | |||
| 80cf1588bb | |||
| ef7d1b29f4 | |||
| 7e8c6884db | |||
| 1dd41e608b | |||
| e41c3e5a05 | |||
| f473280bf5 | |||
| b08fdd6e6b | |||
| f73bfc63be | |||
| f6e7c1eca9 | |||
| 6996d178ef | |||
| 7f7229b199 | |||
| 2d5bd26c36 | |||
| fba150b55b | |||
| 2e7bf58acf | |||
| b12c35f957 | |||
| bcf1b84dc4 | |||
| 7c25e3b46d | |||
| a20381e343 | |||
| bd92610a98 | |||
| e3ee881db6 | |||
| b9eabca464 | |||
| fcc72192f5 | |||
| 5a0696ce64 | |||
| f4a59ff117 | |||
| 129c82c863 | |||
| 968c654320 | |||
| 98bb6c568f | |||
| 72168aa8fa | |||
| 28dde5b9b1 | |||
| c2e0ba200c | |||
| 2eb23ea7ea | |||
| 9906f71c5d | |||
| 593c0d9abc | |||
| 53dfd60b4c | |||
| 3299daf061 | |||
| 2856ca1d98 | |||
| e04b7e11da | |||
| 59dc5ecd1e | |||
| ad6c5ab803 | |||
| 592d1f04c5 | |||
| d76c0b0273 | |||
| 0914391a2b | |||
| d88c1cbb4f | |||
| 35a5d59cbe | |||
| 341e3a0e1c | |||
| a30d1a93dd | |||
| c50d274ae1 | |||
| df19d16269 | |||
| 35af6720ef | |||
| a33432d5de | |||
| 7152eb03de | |||
| 8e5ba8de7f | |||
| 7c10fda451 | |||
| ca88f92b1a | |||
| 5b5f119a65 | |||
| 8caf575946 | |||
| d644d390a7 | |||
| b56dc50e50 | |||
| 63223329dd | |||
| c74992ea85 | |||
| c0c024dcfd | |||
| 0023e27110 | |||
| 882b4ec871 |
@@ -0,0 +1,29 @@
|
||||
name: runner nix smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- .gitea/workflows/runner-nix-smoke.yaml
|
||||
- flake.lock
|
||||
- flake.nix
|
||||
- infra/gitea-runners/**
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
name: nix label smoke
|
||||
runs-on: nix
|
||||
steps:
|
||||
- name: Nix version and cache configuration
|
||||
run: |
|
||||
set -eu
|
||||
nix --version
|
||||
nix config show substituters
|
||||
nix config show trusted-public-keys
|
||||
|
||||
- name: Repository flake evaluation
|
||||
run: |
|
||||
set -eu
|
||||
nix flake check --no-build
|
||||
@@ -0,0 +1,31 @@
|
||||
name: runner ubuntu smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- .gitea/workflows/runner-ubuntu-smoke.yaml
|
||||
- infra/gitea-runners/**
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
name: ubuntu-latest label smoke
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Basic runner information
|
||||
run: |
|
||||
set -eu
|
||||
echo "hello from gitea runner"
|
||||
uname -a
|
||||
|
||||
- name: Docker smoke when available
|
||||
run: |
|
||||
set -eu
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
|
||||
docker run --rm hello-world
|
||||
else
|
||||
echo "docker command not available; skipping Docker smoke"
|
||||
fi
|
||||
@@ -47,6 +47,15 @@ creation_rules:
|
||||
- *hectic-lab-server
|
||||
- *umbriel-bfs
|
||||
|
||||
- path_regex: sus/gitea-runners.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- *nrv
|
||||
- *yukkop
|
||||
- *yukkop-alt
|
||||
- *hectic-lab-server
|
||||
- *umbriel-bfs
|
||||
|
||||
- path_regex: sus/matrix-cluster.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
flake,
|
||||
self,
|
||||
inputs,
|
||||
system ? "aarch64-darwin",
|
||||
...
|
||||
}: let
|
||||
name = builtins.baseNameOf ./.;
|
||||
in inputs.nix-darwin.lib.darwinSystem {
|
||||
inherit system;
|
||||
specialArgs = { inherit flake self inputs; };
|
||||
modules = [
|
||||
inputs.home-manager.darwinModules.home-manager
|
||||
{
|
||||
networking.hostName = name;
|
||||
nixpkgs.hostPlatform = system;
|
||||
nixpkgs.overlays = [ self.overlays.default ];
|
||||
}
|
||||
./${name}.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
{
|
||||
flake,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
name = "yukkop";
|
||||
in {
|
||||
system.primaryUser = name;
|
||||
nix.settings.experimental-features = "nix-command flakes";
|
||||
|
||||
programs.zsh.enable = true;
|
||||
|
||||
services.openssh.enable = true;
|
||||
|
||||
users.users.${name} = {
|
||||
home = "/Users/${name}";
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
aerospace
|
||||
git
|
||||
moreutils
|
||||
neovim
|
||||
tmux
|
||||
];
|
||||
|
||||
launchd.user.agents.aerospace = {
|
||||
serviceConfig = {
|
||||
ProgramArguments = [
|
||||
"${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
|
||||
];
|
||||
RunAtLoad = true;
|
||||
KeepAlive = true;
|
||||
StandardOutPath = "/tmp/aerospace.out.log";
|
||||
StandardErrorPath = "/tmp/aerospace.err.log";
|
||||
};
|
||||
};
|
||||
|
||||
home-manager.useGlobalPkgs = true;
|
||||
home-manager.useUserPackages = true;
|
||||
home-manager.backupFileExtension = "backup";
|
||||
home-manager.sharedModules = [
|
||||
(flake + "/home/module/program/tmux.nix")
|
||||
];
|
||||
home-manager.users.${name} = {
|
||||
home.stateVersion = "25.11";
|
||||
|
||||
home.packages = with pkgs; [
|
||||
iproute2mac
|
||||
jujutsu
|
||||
ripgrep
|
||||
];
|
||||
|
||||
programs.git = {
|
||||
enable = true;
|
||||
lfs.enable = true;
|
||||
settings = {
|
||||
user.name = name;
|
||||
user.email = "hectic.yukkop@gmail.com";
|
||||
push.autoSetupRemote = true;
|
||||
init.defaultBranch = "master";
|
||||
};
|
||||
};
|
||||
|
||||
programs.zsh = {
|
||||
enable = true;
|
||||
enableCompletion = true;
|
||||
autosuggestion.enable = true;
|
||||
syntaxHighlighting.enable = true;
|
||||
|
||||
history = {
|
||||
size = 10000;
|
||||
path = "$HOME/.zsh/.zsh_history";
|
||||
};
|
||||
|
||||
shellAliases = {
|
||||
drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
|
||||
nv = "nvim";
|
||||
tmux = "tmux a";
|
||||
};
|
||||
|
||||
initContent = ''
|
||||
export PATH=/Users/yukkop/.opencode/bin:$PATH
|
||||
'';
|
||||
};
|
||||
|
||||
xdg.configFile."aerospace/aerospace.toml".text = ''
|
||||
start-at-login = false
|
||||
|
||||
enable-normalization-flatten-containers = true
|
||||
enable-normalization-opposite-orientation-for-nested-containers = true
|
||||
|
||||
default-root-container-layout = 'tiles'
|
||||
default-root-container-orientation = 'auto'
|
||||
accordion-padding = 30
|
||||
|
||||
on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
|
||||
automatically-unhide-macos-hidden-apps = false
|
||||
|
||||
[exec]
|
||||
inherit-env-vars = true
|
||||
|
||||
[exec.env-vars]
|
||||
PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
|
||||
|
||||
[gaps]
|
||||
inner.horizontal = 8
|
||||
inner.vertical = 8
|
||||
outer.left = 8
|
||||
outer.bottom = 8
|
||||
outer.top = 8
|
||||
outer.right = 8
|
||||
|
||||
[mode.main.binding]
|
||||
alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
|
||||
|
||||
alt-slash = 'layout tiles horizontal vertical'
|
||||
alt-comma = 'layout accordion horizontal vertical'
|
||||
alt-f = 'fullscreen'
|
||||
|
||||
alt-h = 'focus left'
|
||||
alt-j = 'focus down'
|
||||
alt-k = 'focus up'
|
||||
alt-l = 'focus right'
|
||||
|
||||
alt-shift-h = 'move left'
|
||||
alt-shift-j = 'move down'
|
||||
alt-shift-k = 'move up'
|
||||
alt-shift-l = 'move right'
|
||||
|
||||
alt-minus = 'resize smart -50'
|
||||
alt-equal = 'resize smart +50'
|
||||
|
||||
alt-1 = 'workspace 1'
|
||||
alt-2 = 'workspace 2'
|
||||
alt-3 = 'workspace 3'
|
||||
alt-4 = 'workspace 4'
|
||||
alt-5 = 'workspace 5'
|
||||
alt-6 = 'workspace 6'
|
||||
alt-7 = 'workspace 7'
|
||||
alt-8 = 'workspace 8'
|
||||
alt-9 = 'workspace 9'
|
||||
|
||||
alt-shift-1 = 'move-node-to-workspace 1'
|
||||
alt-shift-2 = 'move-node-to-workspace 2'
|
||||
alt-shift-3 = 'move-node-to-workspace 3'
|
||||
alt-shift-4 = 'move-node-to-workspace 4'
|
||||
alt-shift-5 = 'move-node-to-workspace 5'
|
||||
alt-shift-6 = 'move-node-to-workspace 6'
|
||||
alt-shift-7 = 'move-node-to-workspace 7'
|
||||
alt-shift-8 = 'move-node-to-workspace 8'
|
||||
alt-shift-9 = 'move-node-to-workspace 9'
|
||||
|
||||
alt-tab = 'workspace-back-and-forth'
|
||||
alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
|
||||
|
||||
alt-shift-semicolon = 'mode service'
|
||||
|
||||
[mode.service.binding]
|
||||
esc = ['reload-config', 'mode main']
|
||||
r = ['flatten-workspace-tree', 'mode main']
|
||||
f = ['layout floating tiling', 'mode main']
|
||||
b = ['balance-sizes', 'mode main']
|
||||
backspace = ['close-all-windows-but-current', 'mode main']
|
||||
|
||||
alt-shift-h = ['join-with left', 'mode main']
|
||||
alt-shift-j = ['join-with down', 'mode main']
|
||||
alt-shift-k = ['join-with up', 'mode main']
|
||||
alt-shift-l = ['join-with right', 'mode main']
|
||||
'';
|
||||
};
|
||||
system.stateVersion = 6;
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
haskell = import ./haskell.nix { inherit self system pkgs; };
|
||||
neuro = import ./neuro.nix { inherit self system pkgs; };
|
||||
xmpp = import ./xmpp.nix { inherit self system pkgs; };
|
||||
gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
|
||||
default = pkgs.mkShell {
|
||||
buildInputs =
|
||||
(with self.packages.${system}; [
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
{ pkgs, ... }: let
|
||||
opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
|
||||
|
||||
tofu = pkgs.writeShellScriptBin "tofu" ''
|
||||
exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
|
||||
'';
|
||||
|
||||
giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
|
||||
cat <<'EOF'
|
||||
Gitea runners setup checklist
|
||||
|
||||
Tools available in this shell:
|
||||
tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
|
||||
docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
|
||||
|
||||
Environment expected before real deploy/apply:
|
||||
TF_VAR_hcloud_token
|
||||
TF_VAR_ssh_public_key
|
||||
TF_VAR_ssh_private_key
|
||||
S3 backend credentials and endpoint access
|
||||
a matching SOPS age identity for sus/gitea-runners.yaml
|
||||
kubectl access to the target cluster
|
||||
a concrete registry digest for the pushed Nix-capable runner image if enabling
|
||||
the nix label
|
||||
|
||||
OpenTofu validation gate:
|
||||
tofu version
|
||||
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||
|
||||
Nix image build/publish/digest gate:
|
||||
nix build .#gitea-runner-nix-image
|
||||
publish the archive, then pin the registry-reported digest in the runner label
|
||||
mapping
|
||||
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||
|
||||
SOPS token Secret creation gate:
|
||||
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||
umask 077
|
||||
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||
trap 'rm -f "$token_file"' EXIT
|
||||
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||
--from-file=token="$token_file" \
|
||||
--dry-run=client \
|
||||
-o yaml | kubectl -n gitea-runners apply -f -
|
||||
|
||||
Cluster provision gate:
|
||||
tofu -chdir=infra/gitea-runners/opentofu init
|
||||
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||
|
||||
Kubernetes apply gate:
|
||||
kubectl config current-context
|
||||
kubectl get nodes -o wide
|
||||
kubectl get sc
|
||||
kubectl apply -k infra/gitea-runners/k8s
|
||||
|
||||
Verification commands:
|
||||
kubectl -n gitea-runners get statefulset gitea-runner
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||
|
||||
Main blockers and gates:
|
||||
do not run tofu apply without all external inputs
|
||||
do not apply the k8s overlay until the gitea-runner-token Secret exists
|
||||
do not enable the nix label until the image has been published with a concrete digest
|
||||
do not print, load, or require secrets on shell entry
|
||||
EOF
|
||||
'';
|
||||
in pkgs.mkShell {
|
||||
name = "gitea-runners";
|
||||
|
||||
buildInputs = [
|
||||
tofu
|
||||
giteaRunnersSetup
|
||||
pkgs.nix
|
||||
pkgs.kubectl
|
||||
pkgs.kustomize
|
||||
pkgs.kubeconform
|
||||
pkgs.sops
|
||||
pkgs.age
|
||||
pkgs.awscli2
|
||||
pkgs.hcloud
|
||||
pkgs.tea
|
||||
pkgs.docker
|
||||
pkgs.skopeo
|
||||
pkgs.go-containerregistry
|
||||
pkgs.jq
|
||||
pkgs.yq-go
|
||||
pkgs.curl
|
||||
pkgs.git
|
||||
pkgs.openssh
|
||||
];
|
||||
|
||||
shellHook = ''
|
||||
export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
|
||||
export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
|
||||
export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
|
||||
export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
|
||||
export GITEA_RUNNERS_NAMESPACE="gitea-runners"
|
||||
|
||||
alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
|
||||
alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
|
||||
alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
|
||||
|
||||
echo ""
|
||||
echo "=== Gitea runner setup DevShell ==="
|
||||
echo ""
|
||||
echo "Run gitea-runners-setup for the full setup checklist."
|
||||
echo "Paths: "
|
||||
echo " root=$GITEA_RUNNERS_ROOT"
|
||||
echo " tofu=$GITEA_RUNNERS_TOFU_DIR"
|
||||
echo " k8s=$GITEA_RUNNERS_K8S_DIR"
|
||||
echo " image=$GITEA_RUNNERS_IMAGE_DIR"
|
||||
echo ""
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# Documentation
|
||||
|
||||
- [Using the `hectic` Attic Cache](./attic-cache.md)
|
||||
@@ -0,0 +1,237 @@
|
||||
# Using the `hectic` Attic Cache
|
||||
|
||||
This document explains how to:
|
||||
|
||||
1. pull build artifacts from the cache
|
||||
2. push new artifacts to the cache
|
||||
3. configure this flake to use the cache
|
||||
|
||||
## Cache endpoints
|
||||
|
||||
- API endpoint: `https://cache.hectic-lab.com`
|
||||
- Binary cache endpoint: `https://cache.hectic-lab.com/hectic`
|
||||
|
||||
The `hectic` cache is:
|
||||
|
||||
- public for reads
|
||||
- private for pushes
|
||||
|
||||
## Requirements
|
||||
|
||||
Use the Attic client package:
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client
|
||||
```
|
||||
|
||||
Or run commands directly with:
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c <command>
|
||||
```
|
||||
|
||||
## Read from the cache
|
||||
|
||||
### Get the cache public key
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic cache info hectic
|
||||
```
|
||||
|
||||
Copy the `Public Key` value, which looks like:
|
||||
|
||||
```text
|
||||
hectic:...
|
||||
```
|
||||
|
||||
### Configure Nix to trust the cache
|
||||
|
||||
Per-user: `~/.config/nix/nix.conf`
|
||||
|
||||
```ini
|
||||
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
|
||||
```
|
||||
|
||||
System-wide: `/etc/nix/nix.conf`
|
||||
|
||||
```ini
|
||||
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
|
||||
```
|
||||
|
||||
After that, normal Nix commands can download from the cache automatically:
|
||||
|
||||
```sh
|
||||
nix build .#migrator
|
||||
nix develop
|
||||
nix flake check
|
||||
```
|
||||
|
||||
## Use the cache from this flake
|
||||
|
||||
You can also advertise the cache from `flake.nix`:
|
||||
|
||||
```nix
|
||||
nixConfig = {
|
||||
extra-substituters = [
|
||||
"https://cache.nixos.org"
|
||||
"https://cache.hectic-lab.com/hectic"
|
||||
];
|
||||
extra-trusted-public-keys = [
|
||||
"hectic:PASTE_PUBLIC_KEY_HERE"
|
||||
];
|
||||
};
|
||||
```
|
||||
|
||||
Then users can run:
|
||||
|
||||
```sh
|
||||
nix build --accept-flake-config .#migrator
|
||||
```
|
||||
|
||||
## Log in for pushing
|
||||
|
||||
Pushing requires an Attic token.
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
|
||||
```
|
||||
|
||||
Example with `pass`:
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "$(pass show atticd/hectic-lab/token)"
|
||||
```
|
||||
|
||||
## Push build results
|
||||
|
||||
### Push a package
|
||||
|
||||
```sh
|
||||
nix build .#migrator
|
||||
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||
```
|
||||
|
||||
### Push a check
|
||||
|
||||
```sh
|
||||
nix build .#checks.x86_64-linux.arguments
|
||||
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||
```
|
||||
|
||||
### Push a NixOS system build
|
||||
|
||||
```sh
|
||||
nix build '.#nixosConfigurations."hectic-lab|x86_64-linux".config.system.build.toplevel'
|
||||
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||
```
|
||||
|
||||
## Recommended workflow
|
||||
|
||||
### Local development
|
||||
|
||||
Use the cache for reads only:
|
||||
|
||||
```sh
|
||||
nix build .#migrator
|
||||
nix develop
|
||||
nix flake check
|
||||
```
|
||||
|
||||
### CI / builder
|
||||
|
||||
1. Build
|
||||
2. Push to Attic
|
||||
|
||||
Example:
|
||||
|
||||
```sh
|
||||
nix build .#migrator
|
||||
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||
```
|
||||
|
||||
## Useful commands
|
||||
|
||||
### Show cache info
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic cache info hectic
|
||||
```
|
||||
|
||||
### Check login config
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic cache info local:hectic
|
||||
```
|
||||
|
||||
### Re-login with a new token
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<NEW_TOKEN>"
|
||||
```
|
||||
|
||||
## Common issues
|
||||
|
||||
### `flake 'nixpkgs' does not provide attribute 'attic'`
|
||||
|
||||
Use:
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client
|
||||
```
|
||||
|
||||
Not:
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic
|
||||
```
|
||||
|
||||
### `HTTP 413 Payload Too Large`
|
||||
|
||||
This means nginx rejected the upload body size. The server must allow large uploads on the Attic vhost.
|
||||
|
||||
### Push succeeds for some paths but fails for others
|
||||
|
||||
Usually means:
|
||||
|
||||
- nginx body size limit
|
||||
- timeout/reverse proxy issue
|
||||
- bad token permissions
|
||||
|
||||
### Cache pulls do not work
|
||||
|
||||
Check:
|
||||
|
||||
- `substituters`
|
||||
- `trusted-public-keys`
|
||||
- the exact public key from `attic cache info hectic`
|
||||
|
||||
## Notes about retention and storage
|
||||
|
||||
- The cache currently uses Hetzner Object Storage
|
||||
- If no `retention-period` is configured, cached objects do not expire automatically
|
||||
- This is good for long-lived reuse, but storage usage can grow over time
|
||||
|
||||
## Summary
|
||||
|
||||
### Read access
|
||||
|
||||
```sh
|
||||
nix build .#migrator
|
||||
```
|
||||
|
||||
after configuring:
|
||||
|
||||
```ini
|
||||
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
|
||||
```
|
||||
|
||||
### Push access
|
||||
|
||||
```sh
|
||||
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
|
||||
nix build .#migrator
|
||||
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||
```
|
||||
Generated
+36
-31
@@ -326,7 +326,7 @@
|
||||
"hectic-landing": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs-fixed"
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
@@ -346,7 +346,7 @@
|
||||
"home-manager": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs-fixed"
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
@@ -671,15 +671,15 @@
|
||||
"mechabellum-replay-analysis": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs-fixed"
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1779576166,
|
||||
"narHash": "sha256-5bSuXkQs7KdbaYwDTdwUFlqOccVjPI2y42TZVq8lsNg=",
|
||||
"lastModified": 1780905541,
|
||||
"narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "f00295225c0dade61fe18b32262970c2665fb5fe",
|
||||
"revCount": 110,
|
||||
"rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
|
||||
"revCount": 123,
|
||||
"type": "git",
|
||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||
},
|
||||
@@ -688,6 +688,27 @@
|
||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||
}
|
||||
},
|
||||
"nix-darwin": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1772129556,
|
||||
"narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
|
||||
"owner": "nix-darwin",
|
||||
"repo": "nix-darwin",
|
||||
"rev": "ebec37af18215214173c98cf6356d0aca24a2585",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-darwin",
|
||||
"ref": "nix-darwin-25.11",
|
||||
"repo": "nix-darwin",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-minecraft": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat_3",
|
||||
@@ -872,29 +893,13 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-fixed": {
|
||||
"locked": {
|
||||
"lastModified": 1771419570,
|
||||
"narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-25.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1771419570,
|
||||
"narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
|
||||
"lastModified": 1779796641,
|
||||
"narHash": "sha256-ZsIrKmhp4vbBXoXXmR/tBXA/UCsAQiJL9vsgZEduhVY=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
|
||||
"rev": "25f538306313eae3927264466c70d7001dcea1df",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -908,7 +913,7 @@
|
||||
"inputs": {
|
||||
"flake-parts": "flake-parts_2",
|
||||
"nixpkgs": [
|
||||
"nixpkgs-fixed"
|
||||
"nixpkgs"
|
||||
],
|
||||
"nuschtosSearch": "nuschtosSearch",
|
||||
"systems": "systems_5"
|
||||
@@ -983,13 +988,13 @@
|
||||
"hyprland": "hyprland",
|
||||
"impermanence": "impermanence",
|
||||
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
|
||||
"nix-darwin": "nix-darwin",
|
||||
"nix-minecraft": "nix-minecraft",
|
||||
"nixos-anywhere": "nixos-anywhere",
|
||||
"nixos-hardware": "nixos-hardware",
|
||||
"nixos-mailserver": "nixos-mailserver",
|
||||
"nixos-wsl": "nixos-wsl",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"nixpkgs-fixed": "nixpkgs-fixed",
|
||||
"nixvim": "nixvim",
|
||||
"rust-overlay": "rust-overlay",
|
||||
"sops-nix": "sops-nix"
|
||||
@@ -1002,11 +1007,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1738290352,
|
||||
"narHash": "sha256-YKOHUmc0Clm4tMV8grnxYL4IIwtjTayoq/3nqk0QM7k=",
|
||||
"lastModified": 1780629589,
|
||||
"narHash": "sha256-oHysjxZdaEqkmyDpN8G1bl3V+r9uyRD1O66bH0bq0Cs=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "b031b584125d33d23a0182f91ddbaf3ab4880236",
|
||||
"rev": "7a5a1c0a5cb86a28224304309b68f050835fd1f6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -1,8 +1,18 @@
|
||||
{
|
||||
description = "yukkop's nix utilities";
|
||||
|
||||
nixConfig = {
|
||||
extra-substituters = [
|
||||
"https://cache.nixos.org"
|
||||
"https://cache.hectic-lab.com/hectic"
|
||||
];
|
||||
extra-trusted-public-keys = [
|
||||
"hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
|
||||
];
|
||||
};
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
|
||||
nixpkgs-fixed.url = "github:NixOS/nixpkgs/nixos-25.11";
|
||||
rust-overlay = {
|
||||
url = "github:oxalica/rust-overlay";
|
||||
inputs = {
|
||||
@@ -19,7 +29,7 @@
|
||||
};
|
||||
nixvim = {
|
||||
url = "github:nix-community/nixvim/nixos-25.11";
|
||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
disko = {
|
||||
url = "github:nix-community/disko";
|
||||
@@ -30,7 +40,11 @@
|
||||
};
|
||||
home-manager = {
|
||||
url = "github:nix-community/home-manager/release-25.11";
|
||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
nix-darwin = {
|
||||
url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
nixos-wsl = {
|
||||
url = "github:nix-community/NixOS-WSL";
|
||||
@@ -56,13 +70,13 @@
|
||||
# NOTE(yukkop): private repo - SSH access required.
|
||||
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
||||
url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
mechabellum-replay-analysis = {
|
||||
# NOTE(yukkop): private repo - SSH access required.
|
||||
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
||||
url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
|
||||
@@ -110,7 +124,11 @@
|
||||
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
|
||||
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
|
||||
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
|
||||
"tenix|x86_64-linux" = import ./nixos/system/tenix { inherit flake self inputs; system = "x86_64-linux"; };
|
||||
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
|
||||
};
|
||||
darwinConfigurations = {
|
||||
"yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{ pkgs, ... }: {
|
||||
programs.tmux = {
|
||||
enable = true;
|
||||
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
|
||||
keyMode = "vi";
|
||||
escapeTime = 500;
|
||||
historyLimit = 50000;
|
||||
newSession = true;
|
||||
extraConfig = ''
|
||||
# resurrect
|
||||
set -g @resurrect-strategy-vim 'session'
|
||||
set -g @resurrect-strategy-nvim 'session'
|
||||
set -g @resurrect-capture-pane-contents 'on'
|
||||
|
||||
resurrect_dir="$HOME/.tmux/resurrect"
|
||||
set -g @resurrect-dir $resurrect_dir
|
||||
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
|
||||
|
||||
# continuum
|
||||
set -g @continuum-restore 'on'
|
||||
set -g @continuum-boot 'on'
|
||||
set -g @continuum-save-interval '10'
|
||||
|
||||
bind-key -T copy-mode-vi v send-keys -X begin-selection
|
||||
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
|
||||
|
||||
bind-key O select-pane -t :.-
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
# Gitea runner Nix image
|
||||
|
||||
The repo-owned Nix-capable job image is built by the flake package
|
||||
`gitea-runner-nix-image`.
|
||||
|
||||
```sh
|
||||
nix build .#gitea-runner-nix-image
|
||||
```
|
||||
|
||||
The package emits a Docker archive with the local build tag:
|
||||
|
||||
```text
|
||||
gitea-runner-nix-image:2026-06-07
|
||||
```
|
||||
|
||||
That tag is build metadata only. Do not use it as the final Gitea runner label
|
||||
mapping because runner job images must be immutable.
|
||||
|
||||
## Publication target
|
||||
|
||||
Preferred registry:
|
||||
|
||||
```text
|
||||
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
|
||||
```
|
||||
|
||||
Publish the archive without adding secrets to the image layers, then use the
|
||||
registry-reported digest as the only final `nix` label image reference:
|
||||
|
||||
```text
|
||||
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||
```
|
||||
|
||||
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
|
||||
runner label mapping must use the `@sha256:` reference above. Keep
|
||||
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
|
||||
configuration task explicitly changes it. Only the `nix` label should select
|
||||
this custom image.
|
||||
|
||||
If the Gitea container registry is unavailable, select a private registry that
|
||||
is reachable from the runner Kubernetes cluster and requires authentication that
|
||||
can be provided through Kubernetes image-pull secrets. Record the selected
|
||||
registry and replace the host in the same digest-pinned form:
|
||||
|
||||
```text
|
||||
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
|
||||
```
|
||||
|
||||
Do not fall back to `latest` or a tag-only mapping.
|
||||
|
||||
## Task 7 publication status
|
||||
|
||||
Local build evidence is recorded in
|
||||
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
|
||||
load and tag the image, but pushing to the preferred registry failed with
|
||||
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
|
||||
concrete final mapping. Kubernetes pull smoke is recorded in
|
||||
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
|
||||
is not installed or not on `PATH`.
|
||||
|
||||
Once registry credentials are available, rerun the push, capture the
|
||||
registry-reported digest, and replace `<registry-digest>` in the mapping above
|
||||
before Task 6/9 consumes the label configuration.
|
||||
|
||||
## Image contents
|
||||
|
||||
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
|
||||
`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
|
||||
top-level `flake.nix`:
|
||||
|
||||
```text
|
||||
experimental-features = nix-command flakes
|
||||
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||
sandbox = false
|
||||
```
|
||||
|
||||
No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
|
||||
credential belongs in this image. Runtime secrets stay with the Kubernetes
|
||||
runner configuration and token-file mount contract.
|
||||
@@ -0,0 +1,154 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: gitea-runner-lifecycle
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: gitea-runner-lifecycle
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- statefulsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: gitea-runner-lifecycle
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: gitea-runner-lifecycle
|
||||
namespace: gitea-runners
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: gitea-runner-lifecycle
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: gitea-runner-lifecycle
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
data:
|
||||
cleanup-dry-run.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
namespace="${RUNNER_NAMESPACE:-gitea-runners}"
|
||||
mode="${CLEANUP_MODE:-dry-run}"
|
||||
selector="app.kubernetes.io/name=gitea-runner"
|
||||
|
||||
if [ "$namespace" != "gitea-runners" ]; then
|
||||
printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
|
||||
exit 13
|
||||
fi
|
||||
|
||||
if [ "$mode" != "dry-run" ]; then
|
||||
printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
|
||||
exit 13
|
||||
fi
|
||||
|
||||
printf 'gitea runner lifecycle cleanup dry-run\n'
|
||||
printf 'namespace: %s\n' "$namespace"
|
||||
printf 'mode: %s\n\n' "$mode"
|
||||
|
||||
printf 'StatefulSet:\n'
|
||||
kubectl -n "$namespace" get statefulset gitea-runner -o wide
|
||||
|
||||
printf '\nActive runner pods:\n'
|
||||
kubectl -n "$namespace" get pods -l "$selector" -o wide
|
||||
|
||||
printf '\nRunner /data PVCs:\n'
|
||||
kubectl -n "$namespace" get pvc -l "$selector" -o wide
|
||||
|
||||
printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
|
||||
found_candidate=0
|
||||
for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
pod="${pvc#data-}"
|
||||
if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
|
||||
found_candidate=1
|
||||
printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$found_candidate" -eq 0 ]; then
|
||||
printf 'none\n'
|
||||
fi
|
||||
|
||||
printf '\nGitea registration reconciliation:\n'
|
||||
printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
|
||||
printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: gitea-runner-cleanup-dry-run
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
spec:
|
||||
schedule: "17 3 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
spec:
|
||||
serviceAccountName: gitea-runner-lifecycle
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: cleanup-dry-run
|
||||
image: bitnami/kubectl:1.30
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- /scripts/cleanup-dry-run.sh
|
||||
env:
|
||||
- name: RUNNER_NAMESPACE
|
||||
value: gitea-runners
|
||||
- name: CLEANUP_MODE
|
||||
value: dry-run
|
||||
volumeMounts:
|
||||
- name: lifecycle-scripts
|
||||
mountPath: /scripts
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: lifecycle-scripts
|
||||
configMap:
|
||||
name: gitea-runner-lifecycle
|
||||
defaultMode: 0555
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- service.yaml
|
||||
- service-account.yaml
|
||||
- runner-config.yaml
|
||||
- statefulset.yaml
|
||||
- cleanup-lifecycle.yaml
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: gitea-runner-config
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
data:
|
||||
config.yaml: |
|
||||
log:
|
||||
level: info
|
||||
|
||||
runner:
|
||||
file: /data/.runner
|
||||
capacity: 1
|
||||
envs: {}
|
||||
timeout: 3h
|
||||
insecure: false
|
||||
fetch_timeout: 5s
|
||||
fetch_interval: 2s
|
||||
labels:
|
||||
- ubuntu-latest
|
||||
# The nix label is intentionally disabled until the runner image has a
|
||||
# concrete registry-reported digest; see ../runbook.md before deploy.
|
||||
|
||||
cache:
|
||||
enabled: true
|
||||
dir: /data/cache
|
||||
|
||||
container:
|
||||
network: bridge
|
||||
privileged: false
|
||||
force_pull: true
|
||||
valid_volumes: []
|
||||
docker_host: unix:///runner-docker/docker.sock
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: gitea-runner
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: gitea-runner
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
rules: []
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: gitea-runner
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: gitea-runner
|
||||
namespace: gitea-runners
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: gitea-runner
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: gitea-runner
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
ports:
|
||||
- name: cache
|
||||
port: 8088
|
||||
targetPort: cache
|
||||
@@ -0,0 +1,156 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: gitea-runner
|
||||
namespace: gitea-runners
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
spec:
|
||||
serviceName: gitea-runner
|
||||
replicas: 5
|
||||
podManagementPolicy: Parallel
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
annotations:
|
||||
hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
|
||||
spec:
|
||||
serviceAccountName: gitea-runner
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 60
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: runner
|
||||
image: gitea/act_runner:0.2.11
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: https://gitea.hectic-lab.com
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
||||
value: /runner-secrets/token
|
||||
- name: CONFIG_FILE
|
||||
value: /runner-config/config.yaml
|
||||
- name: DOCKER_HOST
|
||||
value: unix:///runner-docker/docker.sock
|
||||
ports:
|
||||
- name: cache
|
||||
containerPort: 8088
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- test -s /data/.runner && test -S /runner-docker/docker.sock
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- test -S /runner-docker/docker.sock
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: config
|
||||
mountPath: /runner-config
|
||||
readOnly: true
|
||||
- name: runner-token
|
||||
mountPath: /runner-secrets
|
||||
readOnly: true
|
||||
- name: docker-socket
|
||||
mountPath: /runner-docker
|
||||
- name: docker
|
||||
image: docker:27-dind
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --host=unix:///runner-docker/docker.sock
|
||||
- --storage-driver=overlay2
|
||||
- --tls=false
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
- name: DOCKER_HOST
|
||||
value: unix:///runner-docker/docker.sock
|
||||
securityContext:
|
||||
# Privileged rootful DinD is intentionally scoped to this trusted
|
||||
# internal runner pool; never expose it to untrusted fork/PR jobs.
|
||||
privileged: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- docker
|
||||
- info
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- docker
|
||||
- info
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: docker-socket
|
||||
mountPath: /runner-docker
|
||||
- name: docker-graph
|
||||
mountPath: /var/lib/docker
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: gitea-runner-config
|
||||
- name: runner-token
|
||||
secret:
|
||||
secretName: gitea-runner-token
|
||||
items:
|
||||
- key: token
|
||||
path: token
|
||||
defaultMode: 0400
|
||||
- name: docker-socket
|
||||
emptyDir: {}
|
||||
- name: docker-graph
|
||||
emptyDir: {}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea-runner
|
||||
app.kubernetes.io/part-of: gitea-actions
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
@@ -0,0 +1,29 @@
|
||||
# OpenTofu working directory and downloaded modules/providers.
|
||||
.terraform/
|
||||
.terraform.lock.hcl
|
||||
|
||||
# State must live in the S3 backend for production. Local state is allowed only
|
||||
# for throwaway syntax checks with `tofu init -backend=false` and must not be
|
||||
# committed.
|
||||
terraform.tfstate
|
||||
terraform.tfstate.*
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
crash.log
|
||||
crash.*.log
|
||||
|
||||
# Plans can contain secrets or derived infrastructure data.
|
||||
*.tfplan
|
||||
*.plan
|
||||
kubeconfig
|
||||
kubeconfig.yaml
|
||||
*_kubeconfig.yaml
|
||||
|
||||
# Variable files commonly carry credentials. Keep production inputs in SOPS or
|
||||
# external environment/configuration, not in checked-in files.
|
||||
*.tfvars
|
||||
*.tfvars.json
|
||||
override.tf
|
||||
override.tf.json
|
||||
*_override.tf
|
||||
*_override.tf.json
|
||||
@@ -0,0 +1,90 @@
|
||||
# Gitea runner OpenTofu backend contract
|
||||
|
||||
This directory defines the safe backend, provider contract, and kube-hetzner
|
||||
cluster stack for the Gitea runner Kubernetes cluster.
|
||||
|
||||
## Required backend
|
||||
|
||||
Production state must use the OpenTofu S3 backend in `backend.tf`:
|
||||
|
||||
- bucket: `gitea-runner-hectic-lab`
|
||||
- key: `gitea-runners/kube-hetzner/terraform.tfstate`
|
||||
- region: `fsn1`, aligned with the target Hetzner location
|
||||
- encryption: `encrypt = true`
|
||||
- locking: `use_lockfile = true` where the selected S3-compatible endpoint
|
||||
supports it
|
||||
|
||||
Before any production `tofu init`, verify the S3-compatible endpoint, credential
|
||||
source, bucket versioning, encryption behavior, and lockfile support for the
|
||||
chosen object-storage provider. Keep backend authentication externalized through
|
||||
environment variables, AWS-compatible shared config, or the production secret
|
||||
injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
|
||||
tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
|
||||
OpenTofu files.
|
||||
|
||||
## Local state safety
|
||||
|
||||
Production local state is forbidden. Only syntax-only validation/prototyping may
|
||||
use local state, and it must use backend-disabled initialization:
|
||||
|
||||
```sh
|
||||
tofu -chdir=infra/gitea-runners/opentofu init -backend=false
|
||||
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||
```
|
||||
|
||||
Fail the run if production local state appears:
|
||||
|
||||
```sh
|
||||
test ! -e infra/gitea-runners/opentofu/terraform.tfstate
|
||||
test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
|
||||
grep -R 'backend "s3"' infra/gitea-runners/opentofu
|
||||
```
|
||||
|
||||
The `.gitignore` in this directory blocks local state, plans, downloaded
|
||||
providers/modules, and variable files from being committed. Treat any local
|
||||
state file as disposable validation residue, never as production state.
|
||||
|
||||
## Provider and module pins
|
||||
|
||||
`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
|
||||
`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
|
||||
observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
|
||||
module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
|
||||
as locals so Task 5 can wire the module without re-opening the version contract.
|
||||
|
||||
`providers.tf` leaves the `hcloud` provider empty so authentication comes from
|
||||
the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
|
||||
Do not set token values in `.tf` or `.tfvars` files.
|
||||
|
||||
## Cluster shape
|
||||
|
||||
The default cluster is deliberately fixed-size:
|
||||
|
||||
- cluster name: `gitea-runners`
|
||||
- Hetzner location: `fsn1`
|
||||
- private network region: `eu-central`
|
||||
- control plane: one `cpx21` node in pool `control-plane`
|
||||
- workers: three `cpx31` nodes in pool `runner-workers`
|
||||
- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
|
||||
- Longhorn: disabled
|
||||
- autoscaling/KEDA: not enabled in this stack
|
||||
|
||||
The three default workers are sized for the initial five trusted privileged DinD
|
||||
jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
|
||||
`worker_count` to `5` or increase `worker_server_type`, then run a fresh
|
||||
`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
|
||||
|
||||
Required inputs must come from environment or secret injection, for example
|
||||
`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
|
||||
Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
|
||||
kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
|
||||
enabled; this path is ignored as operational secret material.
|
||||
|
||||
## Known state caveat
|
||||
|
||||
kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
|
||||
internal `kube_system_secrets` handling. This task does not claim that risk is
|
||||
solved. Task 5 must verify the generated plan and state before production apply
|
||||
and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
|
||||
keys, and decrypted secrets are absent from committed files and unsafe state
|
||||
evidence.
|
||||
@@ -0,0 +1,16 @@
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "gitea-runner-hectic-lab"
|
||||
key = "gitea-runners/kube-hetzner/terraform.tfstate"
|
||||
region = "fsn1"
|
||||
encrypt = true
|
||||
use_lockfile = true
|
||||
}
|
||||
}
|
||||
|
||||
check "remote_state_contract" {
|
||||
assert {
|
||||
condition = local.production_remote_state
|
||||
error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
locals {
|
||||
default_storage_class = "hcloud-volumes"
|
||||
|
||||
control_plane_nodepools = [
|
||||
{
|
||||
name = "control-plane"
|
||||
server_type = var.control_plane_server_type
|
||||
location = var.hetzner_location
|
||||
labels = []
|
||||
taints = []
|
||||
count = 1
|
||||
},
|
||||
]
|
||||
|
||||
agent_nodepools = [
|
||||
{
|
||||
name = "runner-workers"
|
||||
server_type = var.worker_server_type
|
||||
location = var.hetzner_location
|
||||
labels = ["node-role.hectic-lab/gitea-runner=true"]
|
||||
taints = []
|
||||
count = var.worker_count
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
module "kube_hetzner" {
|
||||
source = "kube-hetzner/kube-hetzner/hcloud"
|
||||
version = "2.19.3"
|
||||
|
||||
providers = {
|
||||
hcloud = hcloud
|
||||
}
|
||||
|
||||
hcloud_token = var.hcloud_token
|
||||
ssh_public_key = var.ssh_public_key
|
||||
ssh_private_key = var.ssh_private_key
|
||||
|
||||
cluster_name = var.cluster_name
|
||||
base_domain = var.base_domain
|
||||
|
||||
# kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
|
||||
# expose that expected path without outputting kubeconfig private key material.
|
||||
create_kubeconfig = true
|
||||
|
||||
network_region = var.network_region
|
||||
load_balancer_location = var.hetzner_location
|
||||
control_plane_nodepools = local.control_plane_nodepools
|
||||
agent_nodepools = local.agent_nodepools
|
||||
|
||||
# Hetzner CSI is the required StorageClass provider for runner PVCs.
|
||||
disable_hetzner_csi = false
|
||||
|
||||
# Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
|
||||
enable_longhorn = false
|
||||
|
||||
# Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
|
||||
# raise worker_count to 5 or increase worker_server_type after validating pod
|
||||
# CPU, memory, and ephemeral-storage pressure in Task 11.
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
output "kubeconfig_path" {
|
||||
description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
|
||||
value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
|
||||
}
|
||||
|
||||
output "cluster_name" {
|
||||
description = "kube-hetzner cluster name."
|
||||
value = var.cluster_name
|
||||
}
|
||||
|
||||
output "node_pool_names" {
|
||||
description = "Control-plane and worker node pool names used by this stack."
|
||||
value = {
|
||||
control_plane = [for pool in local.control_plane_nodepools : pool.name]
|
||||
workers = [for pool in local.agent_nodepools : pool.name]
|
||||
}
|
||||
}
|
||||
|
||||
output "default_storage_class" {
|
||||
description = "Default Hetzner CSI StorageClass expected for runner PVCs."
|
||||
value = local.default_storage_class
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
provider "hcloud" {}
|
||||
@@ -0,0 +1,74 @@
|
||||
variable "hcloud_token" {
|
||||
description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "ssh_public_key" {
|
||||
description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "ssh_private_key" {
|
||||
description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "cluster_name" {
|
||||
description = "Name for the kube-hetzner runner cluster."
|
||||
type = string
|
||||
default = "gitea-runners"
|
||||
|
||||
validation {
|
||||
condition = can(regex("^[a-z0-9-]+$", var.cluster_name))
|
||||
error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
|
||||
}
|
||||
}
|
||||
|
||||
variable "hetzner_location" {
|
||||
description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
|
||||
type = string
|
||||
default = "fsn1"
|
||||
}
|
||||
|
||||
variable "network_region" {
|
||||
description = "Hetzner private network region. eu-central covers fsn1."
|
||||
type = string
|
||||
default = "eu-central"
|
||||
}
|
||||
|
||||
variable "control_plane_server_type" {
|
||||
description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
|
||||
type = string
|
||||
default = "cpx21"
|
||||
}
|
||||
|
||||
variable "worker_server_type" {
|
||||
description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
|
||||
type = string
|
||||
default = "cpx31"
|
||||
}
|
||||
|
||||
variable "worker_count" {
|
||||
description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
|
||||
type = number
|
||||
default = 3
|
||||
|
||||
validation {
|
||||
condition = var.worker_count >= 1
|
||||
error_message = "worker_count must be at least 1."
|
||||
}
|
||||
}
|
||||
|
||||
variable "kubeconfig_path" {
|
||||
description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "base_domain" {
|
||||
description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10.1"
|
||||
|
||||
required_providers {
|
||||
hcloud = {
|
||||
source = "hetznercloud/hcloud"
|
||||
version = "1.60.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud"
|
||||
kube_hetzner_module_version = "2.19.3"
|
||||
hcloud_provider_minimum = ">= 1.59.0"
|
||||
production_remote_state = true
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
# Gitea Runner Infrastructure Runbook
|
||||
|
||||
## Scope
|
||||
|
||||
This directory is the repo-owned boundary for the first Gitea Actions runner
|
||||
pool. Task 1 only establishes the scaffold and immutable decision contract;
|
||||
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
|
||||
and a Nix-capable runner image under the existing subdirectories.
|
||||
|
||||
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
|
||||
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
|
||||
workflow job containers are ephemeral, while each runner pod keeps its runner
|
||||
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||
|
||||
## Immutable decisions
|
||||
|
||||
- Infrastructure is managed with OpenTofu command examples only, using the
|
||||
`tofu` CLI.
|
||||
- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
|
||||
- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
|
||||
- Runner implementation is the non-Enterprise `gitea/runner`.
|
||||
- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
|
||||
- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
|
||||
file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
|
||||
environment variables are not the contract.
|
||||
- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
|
||||
`/data`, including `/data/.runner`.
|
||||
- Container builds run through privileged rootful DinD inside trusted runner
|
||||
pods; host Docker socket mounting is not an implementation path.
|
||||
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
|
||||
the Nix-capable image has been pushed and a concrete registry-reported digest
|
||||
is added to the runner ConfigMap.
|
||||
- First scope is trusted internal workflows only, with no untrusted fork or PR
|
||||
workflow support.
|
||||
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
|
||||
|
||||
## Lifecycle boundaries
|
||||
|
||||
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
|
||||
contract, Hetzner provider configuration, and kube-hetzner module wiring.
|
||||
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
|
||||
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
|
||||
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
|
||||
handoff; package or flake output changes are outside Task 1.
|
||||
- `infra/gitea-runners/runbook.md`: this contract plus later operational
|
||||
commands, rollback notes, and acceptance evidence references.
|
||||
|
||||
## Guardrails
|
||||
|
||||
- Enterprise ARC/actions-runner-controller are rejected alternatives and must
|
||||
not be implemented here. Do not add ARC custom resources, controller install
|
||||
instructions, or GitHub Actions ARC assumptions.
|
||||
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
|
||||
acceptable for trusted internal jobs.
|
||||
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
|
||||
runner pool.
|
||||
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
|
||||
S3 credentials, decrypted SOPS files, or SOPS age keys.
|
||||
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
|
||||
runner token; Kubernetes receives the token as a mounted file secret instead.
|
||||
- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
|
||||
jobs must reach the public HTTPS service.
|
||||
|
||||
## Initial acceptance commands
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```sh
|
||||
test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
|
||||
test -f infra/gitea-runners/runbook.md
|
||||
grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
|
||||
grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
|
||||
grep -R "[t]erraform " infra/gitea-runners || true
|
||||
grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
|
||||
```
|
||||
|
||||
Expected outcomes: the directory and file checks exit 0; the architecture-term
|
||||
grep shows this contract; ARC references appear only in the rejected-alternative
|
||||
guardrail above; there are no forbidden CLI command examples and no unresolved
|
||||
decision placeholders.
|
||||
|
||||
## Downstream placeholders
|
||||
|
||||
- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
|
||||
- Task 3: add SOPS secret contract and runtime token delivery details.
|
||||
- Task 4: define or package the Nix-capable runner image for the `nix` label.
|
||||
- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
|
||||
and document cleanup, rollback, and scaling operations.
|
||||
|
||||
## Runner lifecycle cleanup
|
||||
|
||||
All lifecycle commands are scoped to the runner namespace:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners get statefulset gitea-runner
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
```
|
||||
|
||||
The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
|
||||
runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
|
||||
does not delete pods, PVCs, Docker data, or Gitea runner registrations.
|
||||
|
||||
Run the same inventory on demand without waiting for the schedule:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||
```
|
||||
|
||||
The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
|
||||
manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
|
||||
operator to remove finished jobs manually.
|
||||
|
||||
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||
pod loses `/data/.runner`. That runner identity must then be deregistered from
|
||||
Gitea or the replacement pod must be allowed to re-register intentionally with
|
||||
the current organization runner token. Do not delete an active runner PVC as a
|
||||
normal cleanup step.
|
||||
|
||||
Non-UI Gitea registration reconciliation uses the Gitea API with a separate
|
||||
admin token. Store that token outside this repository and pass it as a file; do
|
||||
not print it:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||
--restart=Never \
|
||||
--image=curlimages/curl:8.10.1 \
|
||||
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||
```
|
||||
|
||||
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
|
||||
has completed and its logs have been collected. Do not keep this admin token in
|
||||
the runner namespace longer than the reconciliation window.
|
||||
|
||||
Only remove a stale Gitea runner registration after the corresponding pod/PVC
|
||||
was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
|
||||
Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
|
||||
cleanup is a fallback, not the only path. Record the removed runner name and the
|
||||
Kubernetes PVC/pod deletion that made it stale.
|
||||
|
||||
After the dry-run list identifies a stale registration and the PVC/pod deletion
|
||||
has been recorded, remove that exact Gitea runner by id through the API:
|
||||
|
||||
```sh
|
||||
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||
umask 077
|
||||
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||
trap 'rm -f "$curl_config"' EXIT
|
||||
{
|
||||
printf 'request = "DELETE"\n'
|
||||
printf 'header = "Authorization: token '
|
||||
cat /secure/path/gitea-admin-token
|
||||
printf '"\n'
|
||||
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||
} > "$curl_config"
|
||||
curl -fsS --config "$curl_config"
|
||||
```
|
||||
|
||||
Do not run the delete command for a runner that still has an active
|
||||
`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
|
||||
being intentionally reset for re-registration.
|
||||
|
||||
## Docker-in-Docker storage cleanup
|
||||
|
||||
Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
|
||||
pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
|
||||
list disk usage before pruning, and run the command only against the `docker`
|
||||
container in runner pods in `gitea-runners`. Because this storage is pod-local,
|
||||
loop over pods for pool-wide cleanup:
|
||||
|
||||
```sh
|
||||
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||
done
|
||||
|
||||
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||
done
|
||||
|
||||
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||
done
|
||||
```
|
||||
|
||||
For one pod, replace the StatefulSet target with the pod name:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
|
||||
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
|
||||
```
|
||||
|
||||
Do not run host-level Docker cleanup commands and do not mount or prune a host
|
||||
Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
|
||||
Kubernetes; the `/data` PVC remains and still controls runner identity.
|
||||
|
||||
## Token rotation
|
||||
|
||||
Rotate the Gitea organization runner token without printing decrypted values:
|
||||
|
||||
```sh
|
||||
sops sus/gitea-runners.yaml
|
||||
umask 077
|
||||
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||
trap 'rm -f "$token_file"' EXIT
|
||||
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||
--from-file=token="$token_file" \
|
||||
--dry-run=client \
|
||||
-o yaml | kubectl -n gitea-runners apply -f -
|
||||
kubectl -n gitea-runners rollout restart statefulset/gitea-runner
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||
```
|
||||
|
||||
The `rollout restart` command above is the controlled restart path for this
|
||||
StatefulSet. Observe the rollout and each ordinal until all replacement pods are
|
||||
Ready; do not delete runner pods directly as part of normal token rotation:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
|
||||
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
|
||||
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
|
||||
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
|
||||
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
```
|
||||
|
||||
Verification must confirm the token file mount remains present while the token
|
||||
value never appears in logs or evidence:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
|
||||
kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||
```
|
||||
|
||||
## Deploy and status
|
||||
|
||||
These commands are executable only when the external inputs are available:
|
||||
|
||||
- `TF_VAR_hcloud_token`
|
||||
- `TF_VAR_ssh_public_key`
|
||||
- `TF_VAR_ssh_private_key`
|
||||
- S3 backend credentials and endpoint access
|
||||
- a matching SOPS age identity for `sus/gitea-runners.yaml`
|
||||
- `kubectl` access to the target cluster
|
||||
- a concrete digest for the pushed Nix-capable runner image, if enabling the
|
||||
`nix` label
|
||||
|
||||
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
|
||||
stale kubeconfig files.
|
||||
|
||||
Before production Kubernetes apply or rollout, satisfy both manifest gates:
|
||||
|
||||
1. Create or update the `gitea-runner-token` Secret from SOPS. The active
|
||||
Kustomize overlay intentionally does not include a placeholder Secret, but
|
||||
the StatefulSet still mounts `secretName: gitea-runner-token` as
|
||||
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
|
||||
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
|
||||
image has been pushed successfully. Enable the `nix` label only by adding a
|
||||
digest-pinned `docker://` mapping with the exact registry-reported sha256
|
||||
digest from that push.
|
||||
|
||||
Use the same SOPS materialization pattern as token rotation before applying the
|
||||
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
|
||||
target namespace; the full overlay remains gated on the Secret and digest
|
||||
decisions:
|
||||
|
||||
```sh
|
||||
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||
umask 077
|
||||
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||
trap 'rm -f "$token_file"' EXIT
|
||||
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||
--from-file=token="$token_file" \
|
||||
--dry-run=client \
|
||||
-o yaml | kubectl -n gitea-runners apply -f -
|
||||
```
|
||||
|
||||
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
|
||||
above succeeds. Do not claim or enable the `nix` runner label until the image
|
||||
publication step has produced the concrete digest.
|
||||
|
||||
```sh
|
||||
tofu -chdir=infra/gitea-runners/opentofu init
|
||||
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||
kubectl config current-context
|
||||
kubectl get nodes -o wide
|
||||
kubectl get sc
|
||||
kubectl apply -k infra/gitea-runners/k8s
|
||||
kubectl -n gitea-runners get statefulset gitea-runner
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||
```
|
||||
|
||||
Expected status after deploy:
|
||||
|
||||
- `kubectl config current-context` names the runner cluster context.
|
||||
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
|
||||
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
|
||||
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
|
||||
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
|
||||
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
|
||||
|
||||
## Scale 5 to 10 to 5
|
||||
|
||||
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
|
||||
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
|
||||
# Run the concurrent smoke workflows now.
|
||||
|
||||
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
```
|
||||
|
||||
After scaling back down, inspect the cleanup dry-run and deregister any stale
|
||||
runner registrations only for pods or PVCs that were intentionally removed.
|
||||
|
||||
## Cleanup and stale runner deregistration
|
||||
|
||||
Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
|
||||
PVC candidates whose pod is gone. It must not delete active resources.
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||
```
|
||||
|
||||
Pool-wide DinD storage checks and cleanup:
|
||||
|
||||
```sh
|
||||
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||
done
|
||||
|
||||
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||
done
|
||||
|
||||
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||
done
|
||||
```
|
||||
|
||||
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||
pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
|
||||
replacement pod re-register intentionally with the current organization token.
|
||||
Never delete an active runner PVC as routine cleanup.
|
||||
|
||||
Non-UI Gitea registration reconciliation uses an admin token stored outside this
|
||||
repository:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||
--restart=Never \
|
||||
--image=curlimages/curl:8.10.1 \
|
||||
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||
```
|
||||
|
||||
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
|
||||
pod has completed and its logs have been collected.
|
||||
|
||||
After the dry-run list identifies a stale registration and the PVC or pod
|
||||
deletion has been recorded, remove that exact Gitea runner by id through the
|
||||
API:
|
||||
|
||||
```sh
|
||||
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||
umask 077
|
||||
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||
trap 'rm -f "$curl_config"' EXIT
|
||||
{
|
||||
printf 'request = "DELETE"\n'
|
||||
printf 'header = "Authorization: token '
|
||||
cat /secure/path/gitea-admin-token
|
||||
printf '"\n'
|
||||
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||
} > "$curl_config"
|
||||
curl -fsS --config "$curl_config"
|
||||
```
|
||||
|
||||
Do not run the delete command for a runner that still has an active
|
||||
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
|
||||
being intentionally reset for re-registration.
|
||||
|
||||
## Application rollback
|
||||
|
||||
Rollback the app layer only. Do not use this section to destroy the cluster.
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners rollout history statefulset/gitea-runner
|
||||
kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||
```
|
||||
|
||||
If a manifest rollback is needed, reapply the repo overlay after checking out the
|
||||
known-good revision, then re-run the rollout checks:
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
|
||||
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||
```
|
||||
|
||||
## Full cluster teardown
|
||||
|
||||
This destroys Hetzner resources owned by the kube-hetzner stack, including the
|
||||
`gitea-runners` cluster nodes, the `control-plane` node pool, the
|
||||
`runner-workers` node pool, the cluster network, load balancer resources,
|
||||
firewall objects, and any attached Hetzner CSI volumes still managed by the
|
||||
stack. Do not run teardown unless the destruction is intentional.
|
||||
|
||||
```sh
|
||||
tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
|
||||
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
|
||||
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
|
||||
```
|
||||
|
||||
## Partial OpenTofu apply recovery
|
||||
|
||||
If `tofu apply` fails after creating some resources, do not destroy blindly.
|
||||
First reconcile state and inspect what the stack thinks exists:
|
||||
|
||||
```sh
|
||||
tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
|
||||
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
|
||||
tofu -chdir=infra/gitea-runners/opentofu state list
|
||||
```
|
||||
|
||||
Then rerun the normal plan path. Use `-target` only as a last resort when a
|
||||
single resource is stuck and the drift is understood.
|
||||
|
||||
## S3 backend recovery
|
||||
|
||||
If backend init or state access fails, first verify the bucket and versioning
|
||||
outside OpenTofu, then reconfigure the backend:
|
||||
|
||||
```sh
|
||||
nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
|
||||
nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
|
||||
tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
|
||||
tofu -chdir=infra/gitea-runners/opentofu plan
|
||||
```
|
||||
|
||||
If the backend reports a stale lock, confirm no `tofu` process is active, then
|
||||
use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
|
||||
unlock a live plan or apply.
|
||||
|
||||
## Gitea outage troubleshooting
|
||||
|
||||
Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
|
||||
containers.
|
||||
|
||||
```sh
|
||||
kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
|
||||
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
|
||||
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||
```
|
||||
|
||||
If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
|
||||
`/data/.runner` just because the service is unavailable. Once Gitea returns,
|
||||
repeat the token rotation or re-registration path if a pod restarted while the
|
||||
service was unavailable and lost its runner identity.
|
||||
|
||||
## Release checklist
|
||||
|
||||
Do not release unless the following evidence files exist and are readable:
|
||||
|
||||
- `.sisyphus/evidence/task-5-cluster-plan.txt`
|
||||
- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
|
||||
- `.sisyphus/evidence/task-9-deploy.txt`
|
||||
- `.sisyphus/evidence/task-9-secret-mount.txt`
|
||||
- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
|
||||
- `.sisyphus/evidence/task-10-nix-workflow.txt`
|
||||
- `.sisyphus/evidence/task-11-scale.txt`
|
||||
- `.sisyphus/evidence/task-11-restart-cleanup.txt`
|
||||
|
||||
If any evidence file is missing, stop and collect it before treating the runbook
|
||||
as complete.
|
||||
+71
-12
@@ -26,12 +26,39 @@
|
||||
"aarch64-darwin"
|
||||
];
|
||||
|
||||
cudaUnfreeNames = [
|
||||
"cuda_nvcc"
|
||||
"cuda_cudart"
|
||||
"cuda_cuobjdump"
|
||||
"cuda_cupti"
|
||||
"cuda_nvdisasm"
|
||||
"cuda_cccl"
|
||||
"cuda_nvml_dev"
|
||||
"cuda_nvrtc"
|
||||
"cuda_nvtx"
|
||||
"cuda_profiler_api"
|
||||
|
||||
"libcusparse_lt"
|
||||
"libcublas"
|
||||
"libcufft"
|
||||
"libcufile"
|
||||
"libcurand"
|
||||
"libcusolver"
|
||||
"libnvjitlink"
|
||||
"libcusparse"
|
||||
"cudnn"
|
||||
];
|
||||
|
||||
cudaUnfreePredicate = pkg:
|
||||
builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
|
||||
|
||||
forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
|
||||
builtins.foldl' (
|
||||
acc: system: let
|
||||
pkgs = import nixpkgs {
|
||||
inherit system;
|
||||
overlays = pkgOverlays;
|
||||
config.allowUnfreePredicate = cudaUnfreePredicate;
|
||||
};
|
||||
systemOutputs = f {
|
||||
system = system;
|
||||
@@ -58,7 +85,7 @@
|
||||
else {};
|
||||
in {
|
||||
# -- For all systems --
|
||||
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems;
|
||||
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
|
||||
|
||||
forSystems = systems: nixpkgs.lib.genAttrs systems;
|
||||
forAllSystems = nixpkgs.lib.genAttrs AllSystems;
|
||||
@@ -67,6 +94,7 @@ in {
|
||||
logs = builtins.readFile ./shell/logs.sh;
|
||||
check-tool = builtins.readFile ./shell/check-tool.sh;
|
||||
local-dir = builtins.readFile ./shell/local-dir.sh;
|
||||
load-sops = builtins.readFile ./shell/load-sops.sh;
|
||||
};
|
||||
|
||||
sharedShellAliases = {
|
||||
@@ -102,7 +130,7 @@ in {
|
||||
|
||||
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
|
||||
# for everything that creates objects in the `hectic` namespace, used by
|
||||
# migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply
|
||||
# migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
|
||||
# the full bundle via lib/hook/apply-hectic-bundle.sh.
|
||||
#
|
||||
# The whole hectic system shares one `versionString`; `hectic-version.sql`
|
||||
@@ -115,19 +143,42 @@ in {
|
||||
hectic = let
|
||||
versionString = lib.fileContents ./hook/sql/HECTIC_VERSION;
|
||||
static = path: { inherit path; sql = builtins.readFile path; };
|
||||
templated = path: {
|
||||
templated = path: let
|
||||
sql = builtins.replaceStrings
|
||||
[ "@HECTIC_VERSION@" ]
|
||||
[ versionString ]
|
||||
(builtins.readFile path);
|
||||
};
|
||||
in {
|
||||
inherit sql;
|
||||
path = builtins.toFile (builtins.baseNameOf (toString path)) sql;
|
||||
};
|
||||
in rec {
|
||||
inherit versionString;
|
||||
version = templated ./hook/sql/hectic-version.sql;
|
||||
secret = static ./hook/sql/hectic-secret.sql;
|
||||
migration = static ./hook/sql/hectic-migration.sql;
|
||||
inheritance = static ./hook/sql/hectic-inheritance.sql;
|
||||
applyBundleScript = ./hook/apply-hectic-bundle.sh;
|
||||
bundleFiles = [
|
||||
version.path
|
||||
secret.path
|
||||
migration.path
|
||||
inheritance.path
|
||||
];
|
||||
applyBundleScript =
|
||||
builtins.replaceStrings
|
||||
[
|
||||
"@HECTIC_VERSION_SQL@"
|
||||
"@HECTIC_SECRET_SQL@"
|
||||
"@HECTIC_MIGRATION_SQL@"
|
||||
"@HECTIC_INHERITANCE_SQL@"
|
||||
]
|
||||
[
|
||||
"${version.path}"
|
||||
"${secret.path}"
|
||||
"${migration.path}"
|
||||
"${inheritance.path}"
|
||||
]
|
||||
(builtins.readFile ./hook/apply-hectic-bundle.sh);
|
||||
};
|
||||
|
||||
# Back-compat alias. Prefer `self.lib.hectic.inheritance`.
|
||||
@@ -164,19 +215,27 @@ in {
|
||||
readModulesRecursive' = path: extraArgs:
|
||||
with lib;
|
||||
with builtins; let
|
||||
paths = pipe "${path}" [
|
||||
(filesystem.listFilesRecursive)
|
||||
(filter (hasSuffix ".nix"))
|
||||
];
|
||||
collectPaths = dir: prefix:
|
||||
concatLists (mapAttrsToList (name: type: let
|
||||
path' = dir + "/${name}";
|
||||
name' = if prefix == "" then name else "${prefix}/${name}";
|
||||
in
|
||||
if type == "directory"
|
||||
then collectPaths path' name'
|
||||
else [{
|
||||
inherit path';
|
||||
name = name';
|
||||
}]
|
||||
) (readDir dir));
|
||||
paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
|
||||
pathToName = flip pipe [
|
||||
(removePrefix "${path}/")
|
||||
(replaceStrings ["/" ".nix"] ["." ""])
|
||||
(removeSuffix ".nix")
|
||||
];
|
||||
attrList =
|
||||
map (path': {
|
||||
name = pathToName (unsafeDiscardStringContext path');
|
||||
value = import path' extraArgs;
|
||||
name = pathToName path'.name;
|
||||
value = import path'.path' extraArgs;
|
||||
})
|
||||
paths;
|
||||
in
|
||||
|
||||
@@ -7,17 +7,12 @@
|
||||
#
|
||||
# Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
|
||||
#
|
||||
# Required env (caller injects from Nix):
|
||||
# HECTIC_VERSION_SQL - path to hectic-version.sql (substituted)
|
||||
# HECTIC_SECRET_SQL - path to hectic-secret.sql
|
||||
# HECTIC_MIGRATION_SQL - path to hectic-migration.sql
|
||||
# HECTIC_INHERITANCE_SQL - path to hectic-inheritance.sql
|
||||
#
|
||||
# Usage:
|
||||
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
||||
#
|
||||
# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via
|
||||
# hectic.load_secrets_from_env() after the bundle is applied.
|
||||
# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
|
||||
# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
|
||||
# SQL file paths are substituted by Nix evaluation time.
|
||||
|
||||
apply_hectic_bundle() {
|
||||
pgurl="${1:-}"
|
||||
@@ -28,29 +23,27 @@ apply_hectic_bundle() {
|
||||
return 3
|
||||
fi
|
||||
|
||||
for var in HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL; do
|
||||
eval "val=\${$var:-}"
|
||||
if [ -z "$val" ]; then
|
||||
printf '%s\n' "apply-hectic-bundle: $var not set" >&2
|
||||
return 3
|
||||
fi
|
||||
if [ ! -r "$val" ]; then
|
||||
printf '%s\n' "apply-hectic-bundle: $var not readable: $val" >&2
|
||||
set -- \
|
||||
"@HECTIC_VERSION_SQL@" \
|
||||
"@HECTIC_SECRET_SQL@" \
|
||||
"@HECTIC_MIGRATION_SQL@" \
|
||||
"@HECTIC_INHERITANCE_SQL@"
|
||||
|
||||
for sql_path do
|
||||
if [ ! -r "$sql_path" ]; then
|
||||
printf '%s\n' "apply-hectic-bundle: SQL file not readable: $sql_path" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_VERSION_SQL" || return 1
|
||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_SECRET_SQL" || return 1
|
||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_MIGRATION_SQL" || return 1
|
||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_INHERITANCE_SQL" || return 1
|
||||
for sql_path do
|
||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$sql_path" || return 1
|
||||
done
|
||||
|
||||
if [ -n "$env_content" ]; then
|
||||
# Dollar-quote with $ps_env$ tag to preserve all content verbatim.
|
||||
env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
|
||||
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
|
||||
SELECT hectic.load_secrets_from_env(\$ps_env\$
|
||||
$env_content
|
||||
\$ps_env\$);
|
||||
SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
|
||||
SQL
|
||||
fi
|
||||
|
||||
|
||||
+23
-25
@@ -4,8 +4,8 @@ Single source of truth for every object created in the `hectic` PostgreSQL
|
||||
schema. Consumed by:
|
||||
|
||||
- `package/migrator` — applies the bundle on `migrator init` (mandatory).
|
||||
- `package/db-tool` — applies the bundle in `database hydrate` (default; opt
|
||||
out with `--no-hook`).
|
||||
- `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
|
||||
(default; opt out with `--no-hook`) and in `db-ops secrets load`.
|
||||
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
|
||||
paths exposed via `self.lib.hectic.*.path`.
|
||||
|
||||
@@ -42,23 +42,26 @@ that already matches.
|
||||
```nix
|
||||
self.lib.hectic = {
|
||||
versionString; # e.g. "0.1.0"
|
||||
version = { sql; }; # templated
|
||||
version = { sql; path; }; # templated
|
||||
secret = { sql; path; };
|
||||
migration = { sql; path; };
|
||||
inheritance = { sql; path; };
|
||||
applyBundleScript; # ./hook/apply-hectic-bundle.sh
|
||||
bundleFiles; # ordered bundle file paths
|
||||
applyBundleScript; # generated helper shell source with paths embedded
|
||||
};
|
||||
```
|
||||
|
||||
`.sql` is the file contents as a string. `.path` is the Nix store path of the
|
||||
verbatim source (only available on non-templated entries; consumers needing a
|
||||
materialized version of `version.sql` must do
|
||||
`pkgs.runCommand "hectic-version.sql" { text = self.lib.hectic.version.sql; passAsFile = ["text"]; } ''cp "$textPath" "$out"''`).
|
||||
materialized file to pass to `psql -f`. `version.path` is generated at Nix
|
||||
evaluation time from the templated SQL; the other `*.path` entries point at the
|
||||
verbatim source files in the store.
|
||||
|
||||
## Shell helper (`apply-hectic-bundle.sh`)
|
||||
|
||||
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper sourced by both
|
||||
`migrator` and `db-tool`. Public entry point:
|
||||
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
|
||||
`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
|
||||
SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
|
||||
shell source directly into their generated scripts. Public entry point:
|
||||
|
||||
```sh
|
||||
apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
||||
@@ -70,28 +73,23 @@ apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
||||
dollar-quoted (`$ps_env$`) string so secret values cannot terminate the
|
||||
literal.
|
||||
|
||||
Required environment (paths to the SQL files):
|
||||
|
||||
- `HECTIC_VERSION_SQL`
|
||||
- `HECTIC_SECRET_SQL`
|
||||
- `HECTIC_MIGRATION_SQL`
|
||||
- `HECTIC_INHERITANCE_SQL`
|
||||
|
||||
`migrator` and `db-tool` set these via Nix at build time. External consumers
|
||||
typically invoke `psql -f` against the paths directly instead of sourcing the
|
||||
helper.
|
||||
The SQL file paths are embedded into the helper at Nix evaluation time, so
|
||||
callers only need to source the generated script and call the function.
|
||||
External consumers that do not want to source the helper can still invoke
|
||||
`psql -f` against `self.lib.hectic.bundleFiles` or the individual
|
||||
`self.lib.hectic.*.path` entries directly.
|
||||
|
||||
## Adding a new SQL file
|
||||
|
||||
1. Add `lib/hook/sql/hectic-<name>.sql`.
|
||||
2. Wire it into `lib/default.nix` under `lib.hectic.<name>`.
|
||||
3. Inject `HECTIC_<NAME>_SQL` in both `package/migrator/default.nix` and
|
||||
`package/db-tool/default.nix`.
|
||||
4. Append a `psql -f "$HECTIC_<NAME>_SQL"` step to
|
||||
`lib/hook/apply-hectic-bundle.sh` in the correct order.
|
||||
3. Add its `.path` to `lib.hectic.bundleFiles` in the correct order.
|
||||
4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
|
||||
update `lib/hook/apply-hectic-bundle.sh` to apply the file.
|
||||
5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
|
||||
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`
|
||||
and `test/package/db-tool/test/postgresql/hydrate-hook/`.
|
||||
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
|
||||
`test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
|
||||
bundle-loading coverage.
|
||||
|
||||
## Versioning
|
||||
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
load_sops_shell_quote() {
|
||||
printf "'"
|
||||
printf '%s' "$1" | sed "s/'/'\"'\"'/g"
|
||||
printf "'"
|
||||
}
|
||||
|
||||
load_sops_normalize_key() {
|
||||
load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
|
||||
|
||||
case "$load_sops_normalized_key" in
|
||||
''|[!A-Z_]*|*[!A-Z0-9_]*)
|
||||
printf 'load-sops: invalid environment name after key normalization\n' >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
printf '%s\n' "$load_sops_normalized_key"
|
||||
}
|
||||
|
||||
load_sops_env_from_sops_file() {
|
||||
load_sops_file=$1
|
||||
load_sops_extract=${2-}
|
||||
|
||||
if ! command -v sops >/dev/null 2>&1; then
|
||||
printf 'load-sops: required tool `sops` not found\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! command -v yq >/dev/null 2>&1; then
|
||||
printf 'load-sops: required tool `yq` not found\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
load_sops_decrypted=''
|
||||
load_sops_attempt=0
|
||||
load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
|
||||
load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
|
||||
|
||||
while :; do
|
||||
if [ -n "$load_sops_extract" ]; then
|
||||
if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
|
||||
load_sops_status=0
|
||||
else
|
||||
load_sops_status=$?
|
||||
fi
|
||||
else
|
||||
if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
|
||||
load_sops_status=0
|
||||
else
|
||||
load_sops_status=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$load_sops_status" -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$load_sops_prompt" != 1 ]; then
|
||||
printf 'load-sops: failed to decrypt file\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
|
||||
printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
load_sops_attempt=$((load_sops_attempt + 1))
|
||||
if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
|
||||
printf 'load-sops: decrypt failed after configured retries\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
|
||||
load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
|
||||
load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
|
||||
case "$load_sops_use_script" in
|
||||
auto)
|
||||
if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
|
||||
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
|
||||
load_sops_script_status=0
|
||||
else
|
||||
load_sops_script_status=$?
|
||||
fi
|
||||
if [ "$load_sops_script_status" -eq 0 ]; then
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
1)
|
||||
if ! command -v script >/dev/null 2>&1; then
|
||||
printf 'load-sops: required tool `script` not found\n' >&2
|
||||
return 1
|
||||
fi
|
||||
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
|
||||
load_sops_script_status=0
|
||||
else
|
||||
load_sops_script_status=$?
|
||||
fi
|
||||
if [ "$load_sops_script_status" -eq 0 ]; then
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
0)
|
||||
;;
|
||||
*)
|
||||
printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
|
||||
if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
|
||||
printf 'load-sops: failed to read prompt input\n' >&2
|
||||
return 1
|
||||
fi
|
||||
export SOPS_AGE_KEY_CMD
|
||||
done
|
||||
|
||||
load_sops_env_from_yaml_text "$load_sops_decrypted"
|
||||
}
|
||||
|
||||
load_sops_env_from_yaml_file() {
|
||||
load_sops_file=$1
|
||||
|
||||
if ! command -v yq >/dev/null 2>&1; then
|
||||
printf 'load-sops: required tool `yq` not found\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
|
||||
}
|
||||
|
||||
load_sops_env_from_yaml_text() {
|
||||
load_sops_yaml=$1
|
||||
load_sops_seen=''
|
||||
|
||||
if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
|
||||
load_sops_keys_status=0
|
||||
else
|
||||
load_sops_keys_status=$?
|
||||
fi
|
||||
|
||||
if [ "$load_sops_keys_status" -ne 0 ]; then
|
||||
printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS= read -r load_sops_key; do
|
||||
[ -n "$load_sops_key" ] || continue
|
||||
|
||||
load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
|
||||
|
||||
case "
|
||||
$load_sops_seen
|
||||
" in
|
||||
*"
|
||||
$load_sops_name
|
||||
"*)
|
||||
if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
|
||||
printf 'load-sops: normalized environment name collision\n' >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
load_sops_seen=${load_sops_seen}${load_sops_seen:+"
|
||||
"}$load_sops_name
|
||||
|
||||
load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
|
||||
printf 'load-sops: failed to inspect YAML value kind\n' >&2
|
||||
return 1
|
||||
}
|
||||
load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
|
||||
printf 'load-sops: failed to inspect YAML value tag\n' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
if [ "$load_sops_kind" != scalar ]; then
|
||||
printf 'load-sops: top-level YAML values must be scalars\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$load_sops_tag" = '!!null' ]; then
|
||||
printf 'load-sops: top-level YAML null values are not supported\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
|
||||
eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
|
||||
if [ -n "$load_sops_already_set" ]; then
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
|
||||
printf 'load-sops: failed to read YAML scalar value\n' >&2
|
||||
return 1
|
||||
}
|
||||
load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
|
||||
eval "export $load_sops_name=$load_sops_quoted"
|
||||
done <<EOF
|
||||
$load_sops_keys
|
||||
EOF
|
||||
}
|
||||
@@ -9,11 +9,11 @@ with self.lib;
|
||||
let
|
||||
# Combine hectic modules into one
|
||||
hectic.imports = attrValues (
|
||||
readModulesRecursive' ./hectic { inherit flake self inputs; }
|
||||
readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
|
||||
);
|
||||
# Read generic modules separately
|
||||
generic = readModulesRecursive'
|
||||
./generic
|
||||
(flake + "/nixos/module/generic")
|
||||
{ inherit flake self inputs; };
|
||||
in generic // {
|
||||
inherit hectic;
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
];
|
||||
|
||||
adminNames = [ "yukkop" ];
|
||||
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||
|
||||
cfg = config.hectic.generic.matrix-cluster;
|
||||
in {
|
||||
@@ -39,7 +40,7 @@ in {
|
||||
value = {
|
||||
key = "matrix/users/${name}/password";
|
||||
owner = "matrix-synapse";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
}) userNames
|
||||
);
|
||||
|
||||
@@ -250,6 +250,7 @@ in {
|
||||
# failover flip does not need a separate provisioning step.
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
|
||||
"Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
|
||||
];
|
||||
|
||||
systemd.services.matrix-cluster-signing-key = {
|
||||
@@ -319,6 +320,12 @@ in {
|
||||
media_store_path = s3Cfg.mediaStorePath;
|
||||
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
|
||||
|
||||
# Tolerate bursty Element/iPhone presence syncs without disabling limits.
|
||||
rc_presence.per_user = {
|
||||
per_second = 0.5;
|
||||
burst_count = 5;
|
||||
};
|
||||
|
||||
experimental_features = {
|
||||
msc3266_enabled = true;
|
||||
msc4140_enabled = true;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
inputs,
|
||||
flake,
|
||||
self,
|
||||
...
|
||||
}: {
|
||||
pkgs,
|
||||
lib,
|
||||
@@ -27,7 +27,15 @@ in {
|
||||
users.defaultUserShell = pkgs.zsh;
|
||||
|
||||
# Enable flakes and new 'nix' command
|
||||
nix.settings.experimental-features = "nix-command flakes";
|
||||
nix.settings = {
|
||||
experimental-features = "nix-command flakes";
|
||||
extra-substituters = [
|
||||
"https://cache.hectic-lab.com/hectic"
|
||||
];
|
||||
extra-trusted-public-keys = [
|
||||
"hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall.enable = true;
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
{
|
||||
inputs,
|
||||
flake,
|
||||
self,
|
||||
...
|
||||
}:
|
||||
{
|
||||
pkgs,
|
||||
@@ -11,6 +9,13 @@
|
||||
}: let
|
||||
cfg = config.hectic.hardware.hetzner-cloud;
|
||||
isNewer = cfg.generation == "newer";
|
||||
networkMatchConfig =
|
||||
(lib.optionalAttrs (cfg.networkMatchConfigName != null) {
|
||||
Name = cfg.networkMatchConfigName;
|
||||
})
|
||||
// (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
|
||||
PermanentMACAddress = cfg.networkMatchConfigMac;
|
||||
});
|
||||
in {
|
||||
options.hectic.hardware.hetzner-cloud = {
|
||||
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
|
||||
@@ -51,6 +56,14 @@ in {
|
||||
|
||||
'';
|
||||
};
|
||||
floatingIpv4 = lib.mkOption {
|
||||
type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
|
||||
default = null;
|
||||
example = "188.243.124.247";
|
||||
description = ''
|
||||
Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
|
||||
'';
|
||||
};
|
||||
device = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
|
||||
@@ -65,14 +78,27 @@ in {
|
||||
'';
|
||||
};
|
||||
networkMatchConfigName = lib.mkOption {
|
||||
type = lib.types.strMatching "^(enp1s0|ens3|eth0)$";
|
||||
type = with lib.types; nullOr str;
|
||||
default = null;
|
||||
example = "enp1s0";
|
||||
description = ''
|
||||
type of network conection,
|
||||
on older hetzner servers may be `ens3` or else
|
||||
on newer probably `enp1s0`
|
||||
Optional interface name to match in systemd-networkd.
|
||||
|
||||
you can use `networkctl list` on server to know it
|
||||
Prefer `networkMatchConfigMac` for stable matching across rescue
|
||||
images and installed systems that may rename interfaces differently.
|
||||
|
||||
You can use `networkctl list` on server to know it.
|
||||
'';
|
||||
};
|
||||
networkMatchConfigMac = lib.mkOption {
|
||||
type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
|
||||
default = null;
|
||||
example = "92:00:08:4a:b0:32";
|
||||
description = ''
|
||||
Optional permanent MAC address to match in systemd-networkd.
|
||||
|
||||
This is the preferred Hetzner Cloud matching method because interface
|
||||
names can differ between rescue images and installed NixOS systems.
|
||||
'';
|
||||
};
|
||||
};
|
||||
@@ -80,6 +106,15 @@ in {
|
||||
config = lib.mkIf cfg.enable (lib.mkMerge
|
||||
[
|
||||
{
|
||||
boot.loader.systemd-boot.enable = false;
|
||||
boot.loader.efi.canTouchEfiVariables = false;
|
||||
boot.loader.grub = {
|
||||
enable = true;
|
||||
efiSupport = true;
|
||||
efiInstallAsRemovable = true;
|
||||
device = "nodev";
|
||||
};
|
||||
|
||||
boot.initrd.availableKernelModules = [
|
||||
"ata_piix"
|
||||
"uhci_hcd"
|
||||
@@ -90,12 +125,12 @@ in {
|
||||
networking.useNetworkd = true;
|
||||
systemd.network.enable = true;
|
||||
systemd.network.networks."30-wan" = {
|
||||
matchConfig.Name = cfg.networkMatchConfigName;
|
||||
matchConfig = networkMatchConfig;
|
||||
networkConfig.DHCP = "no";
|
||||
address = [
|
||||
"${cfg.ipv4}/32"
|
||||
"${cfg.ipv6}::/64"
|
||||
];
|
||||
] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
|
||||
routes = [
|
||||
{ Gateway = "172.31.1.1"; GatewayOnLink = true; }
|
||||
{ Gateway = "fe80::1"; }
|
||||
@@ -137,6 +172,13 @@ in {
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
|
||||
message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
|
||||
}
|
||||
];
|
||||
}
|
||||
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
|
||||
boot.initrd.kernelModules = [ "virtio_gpu" ];
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
{ ... }:
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
cfg = config.hectic.hardware.njalla;
|
||||
in {
|
||||
options.hectic.hardware.njalla = {
|
||||
enable = lib.mkEnableOption "Enable njalla hardware configurations";
|
||||
ipv4 = lib.mkOption {
|
||||
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
|
||||
example = "185.193.126.103";
|
||||
description = ''
|
||||
Njalla IPv4 address assigned to the host.
|
||||
'';
|
||||
};
|
||||
ipv4PrefixLength = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 24;
|
||||
example = 24;
|
||||
description = ''
|
||||
Njalla IPv4 prefix length.
|
||||
'';
|
||||
};
|
||||
ipv4Gateway = lib.mkOption {
|
||||
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
|
||||
default = "185.193.126.1";
|
||||
example = "185.193.126.1";
|
||||
description = ''
|
||||
Njalla IPv4 gateway.
|
||||
'';
|
||||
};
|
||||
ipv6 = lib.mkOption {
|
||||
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
|
||||
example = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
|
||||
description = ''
|
||||
Njalla IPv6 address assigned to the host.
|
||||
'';
|
||||
};
|
||||
ipv6PrefixLength = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 64;
|
||||
example = 64;
|
||||
description = ''
|
||||
Njalla IPv6 prefix length.
|
||||
'';
|
||||
};
|
||||
ipv6Gateway = lib.mkOption {
|
||||
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
|
||||
default = "2a0a:3840:1337:126::1";
|
||||
example = "2a0a:3840:1337:126::1";
|
||||
description = ''
|
||||
Njalla IPv6 gateway.
|
||||
'';
|
||||
};
|
||||
networkMatchConfigName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "eth0";
|
||||
example = "eth0";
|
||||
description = ''
|
||||
Njalla container network interface name.
|
||||
'';
|
||||
};
|
||||
device = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/dev/vda";
|
||||
example = "/dev/disk/by-id/virtio-root";
|
||||
description = ''
|
||||
Njalla installation disk for disko/nixos-anywhere.
|
||||
|
||||
`/dev/vda` is the default block device visible on the inspected Njalla
|
||||
host. Prefer a stable `/dev/disk/by-id/...` path when available.
|
||||
'';
|
||||
};
|
||||
enableDisko = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Whether to provide a disko layout for nixos-anywhere installs.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable (lib.mkMerge [
|
||||
{
|
||||
boot.isContainer = true;
|
||||
|
||||
networking.useDHCP = false;
|
||||
networking.useNetworkd = true;
|
||||
systemd.network.enable = true;
|
||||
systemd.network.networks."30-wan" = {
|
||||
matchConfig.Name = cfg.networkMatchConfigName;
|
||||
networkConfig.DHCP = "no";
|
||||
address = [
|
||||
"${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
|
||||
"${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
|
||||
];
|
||||
routes = [
|
||||
{ Gateway = cfg.ipv4Gateway; }
|
||||
{ Gateway = cfg.ipv6Gateway; }
|
||||
];
|
||||
};
|
||||
|
||||
networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
|
||||
}
|
||||
(lib.mkIf cfg.enableDisko {
|
||||
boot.loader.grub.device = cfg.device;
|
||||
|
||||
disko.devices.disk.main = {
|
||||
type = "disk";
|
||||
device = cfg.device;
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
boot = {
|
||||
size = "1M";
|
||||
type = "EF02";
|
||||
priority = 1;
|
||||
};
|
||||
root = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "ext4";
|
||||
mountpoint = "/";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
})
|
||||
]);
|
||||
}
|
||||
@@ -25,36 +25,7 @@ in {
|
||||
programs.bash.shellAliases.tmux = "tmux a";
|
||||
|
||||
home-manager.sharedModules = [
|
||||
{
|
||||
programs.tmux = {
|
||||
enable = true;
|
||||
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
|
||||
keyMode = "vi";
|
||||
escapeTime = 500;
|
||||
historyLimit = 50000;
|
||||
newSession = true;
|
||||
extraConfig = ''
|
||||
# resurrect
|
||||
set -g @resurrect-strategy-vim 'session'
|
||||
set -g @resurrect-strategy-nvim 'session'
|
||||
set -g @resurrect-capture-pane-contents 'on'
|
||||
|
||||
resurrect_dir="$HOME/.tmux/resurrect"
|
||||
set -g @resurrect-dir $resurrect_dir
|
||||
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
|
||||
|
||||
# continuum
|
||||
set -g @continuum-restore 'on'
|
||||
set -g @continuum-boot 'on'
|
||||
set -g @continuum-save-interval '10'
|
||||
|
||||
bind-key -T copy-mode-vi v send-keys -X begin-selection
|
||||
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
|
||||
|
||||
bind-key O select-pane -t :.-
|
||||
'';
|
||||
};
|
||||
}
|
||||
(flake + "/home/module/program/tmux.nix")
|
||||
];
|
||||
|
||||
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
{ ... }: {
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
cfg = config.hectic.services.attic;
|
||||
in {
|
||||
options.hectic.services.attic = {
|
||||
enable = lib.mkEnableOption "Attic binary cache server";
|
||||
|
||||
hostName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public hostname used by clients to reach this Attic server.";
|
||||
};
|
||||
|
||||
listenAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
description = "Local address atticd binds to behind the reverse proxy.";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8080;
|
||||
description = "Local port atticd binds to behind the reverse proxy.";
|
||||
};
|
||||
|
||||
environmentFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = ''
|
||||
SOPS-backed environment file containing Attic JWT and object-storage
|
||||
credentials.
|
||||
'';
|
||||
};
|
||||
|
||||
storage = {
|
||||
bucket = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Hetzner Object Storage bucket name used by Attic.";
|
||||
};
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "S3-compatible HTTPS endpoint for Hetzner Object Storage.";
|
||||
};
|
||||
|
||||
region = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Region name for Hetzner Object Storage.";
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.atticd = {
|
||||
enable = true;
|
||||
environmentFile = cfg.environmentFile;
|
||||
settings = {
|
||||
listen = "${cfg.listenAddress}:${toString cfg.port}";
|
||||
allowed-hosts = [ cfg.hostName ];
|
||||
api-endpoint = "https://${cfg.hostName}/";
|
||||
compression.type = "zstd";
|
||||
storage = {
|
||||
type = "s3";
|
||||
bucket = cfg.storage.bucket;
|
||||
endpoint = cfg.storage.endpoint;
|
||||
region = cfg.storage.region;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -31,7 +31,7 @@ in {
|
||||
|
||||
locations."= /config.element.${matrixDomain}.json".return = "302 /config.json";
|
||||
|
||||
root = pkgs.element-web.override {
|
||||
root = pkgs.hectic.element-web.override {
|
||||
conf = {
|
||||
default_server_config = {
|
||||
"m.homeserver".base_url = "https://${matrixDomain}";
|
||||
@@ -43,6 +43,8 @@ in {
|
||||
matrixDomain
|
||||
];
|
||||
|
||||
hectic.videoMessages.enabled = true;
|
||||
|
||||
jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) {
|
||||
preferred_domain = jitsiPreferredDomain;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
{ ... }: {
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
cfg = config.hectic.services.ente;
|
||||
|
||||
webHostNames = [
|
||||
cfg.domains.accounts
|
||||
cfg.domains.cast
|
||||
cfg.domains.photos
|
||||
];
|
||||
in {
|
||||
options.hectic.services.ente = {
|
||||
enable = lib.mkEnableOption "Ente Photos self-hosted service";
|
||||
|
||||
apiDomain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public hostname for the Ente Museum API.";
|
||||
};
|
||||
|
||||
domains = {
|
||||
accounts = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public hostname for the Ente accounts web app.";
|
||||
};
|
||||
|
||||
cast = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public hostname for the Ente cast web app.";
|
||||
};
|
||||
|
||||
albums = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public hostname for public Ente album links.";
|
||||
};
|
||||
|
||||
photos = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Public hostname for the Ente Photos web app.";
|
||||
};
|
||||
};
|
||||
|
||||
maxUploadSize = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10G";
|
||||
description = "Maximum request body accepted by nginx in front of Museum.";
|
||||
};
|
||||
|
||||
disableRegistration = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Whether Museum should reject new account registration.";
|
||||
};
|
||||
|
||||
smtp = {
|
||||
enable = lib.mkEnableOption "SMTP delivery for Ente verification emails";
|
||||
|
||||
host = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
description = "SMTP host Museum uses to send verification emails.";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 25;
|
||||
description = "SMTP port Museum uses to send verification emails.";
|
||||
};
|
||||
|
||||
email = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "From email address used by Museum.";
|
||||
};
|
||||
|
||||
senderName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "Ente Photos";
|
||||
description = "Display name used for Ente verification emails.";
|
||||
};
|
||||
|
||||
encryption = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.enum [ "tls" "ssl" ]);
|
||||
default = null;
|
||||
description = "Optional SMTP encryption mode. Leave null for local plaintext SMTP.";
|
||||
};
|
||||
};
|
||||
|
||||
storage = {
|
||||
bucket = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "S3-compatible bucket used by Ente for photo object storage.";
|
||||
};
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "S3-compatible endpoint URL.";
|
||||
};
|
||||
|
||||
region = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "S3-compatible region name.";
|
||||
};
|
||||
|
||||
hotStorage = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"b2-eu-cen"
|
||||
"wasabi-eu-central-2-v3"
|
||||
"scw-eu-fr-v3"
|
||||
];
|
||||
default = "b2-eu-cen";
|
||||
description = ''
|
||||
Museum's primary hot-storage key. Upstream requires one of its
|
||||
historical S3 storage identifiers even when the backing provider is a
|
||||
generic S3-compatible service.
|
||||
'';
|
||||
};
|
||||
|
||||
usePathStyleUrls = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Whether Museum should use path-style S3 URLs.";
|
||||
};
|
||||
};
|
||||
|
||||
secrets = {
|
||||
encryptionKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "File containing Museum key.encryption.";
|
||||
};
|
||||
|
||||
hashKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "File containing Museum key.hash.";
|
||||
};
|
||||
|
||||
jwtSecretFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "File containing Museum jwt.secret.";
|
||||
};
|
||||
|
||||
s3AccessKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "File containing the S3 access key.";
|
||||
};
|
||||
|
||||
s3SecretKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "File containing the S3 secret key.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.ente = {
|
||||
api = {
|
||||
enable = true;
|
||||
enableLocalDB = true;
|
||||
domain = cfg.apiDomain;
|
||||
|
||||
nginx.enable = true;
|
||||
|
||||
settings = {
|
||||
key = {
|
||||
encryption._secret = cfg.secrets.encryptionKeyFile;
|
||||
hash._secret = cfg.secrets.hashKeyFile;
|
||||
};
|
||||
|
||||
jwt.secret._secret = cfg.secrets.jwtSecretFile;
|
||||
|
||||
s3 = {
|
||||
hot_storage.primary = cfg.storage.hotStorage;
|
||||
derived-storage = cfg.storage.hotStorage;
|
||||
are_local_buckets = false;
|
||||
use_path_style_urls = cfg.storage.usePathStyleUrls;
|
||||
|
||||
${cfg.storage.hotStorage} = {
|
||||
key._secret = cfg.secrets.s3AccessKeyFile;
|
||||
secret._secret = cfg.secrets.s3SecretKeyFile;
|
||||
endpoint = cfg.storage.endpoint;
|
||||
region = cfg.storage.region;
|
||||
bucket = cfg.storage.bucket;
|
||||
};
|
||||
};
|
||||
|
||||
internal.disable-registration = cfg.disableRegistration;
|
||||
|
||||
smtp = lib.mkIf cfg.smtp.enable ({
|
||||
inherit (cfg.smtp) host port email;
|
||||
sender-name = cfg.smtp.senderName;
|
||||
} // lib.optionalAttrs (cfg.smtp.encryption != null) {
|
||||
encryption = cfg.smtp.encryption;
|
||||
});
|
||||
};
|
||||
};
|
||||
|
||||
web = {
|
||||
enable = true;
|
||||
domains = {
|
||||
api = cfg.apiDomain;
|
||||
inherit (cfg.domains) accounts cast albums photos;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx.virtualHosts =
|
||||
(lib.genAttrs webHostNames (_: {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
})) // {
|
||||
${cfg.apiDomain} = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
extraConfig = lib.mkForce ''
|
||||
client_max_body_size ${cfg.maxUploadSize};
|
||||
'';
|
||||
locations."/".extraConfig = ''
|
||||
proxy_read_timeout 600s;
|
||||
proxy_send_timeout 600s;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -56,8 +56,18 @@ in {
|
||||
certificateScheme = "acme-nginx";
|
||||
};
|
||||
|
||||
# NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records
|
||||
services.postfix.settings.main.inet_protocols = lib.mkDefault "ipv4";
|
||||
services.postfix.settings.main = {
|
||||
# NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records.
|
||||
inet_protocols = lib.mkDefault "ipv4";
|
||||
|
||||
# NOTE(yukkop): nixos-mailserver enables DANE by default. Some large MXes
|
||||
# currently fail certificate verification under this policy, which leaves
|
||||
# otherwise valid transactional mail deferred in the queue. Keep STARTTLS
|
||||
# opportunistic for outbound delivery rather than blocking mail entirely.
|
||||
smtp_tls_security_level = lib.mkForce "may";
|
||||
smtp_dns_support_level = lib.mkForce "enabled";
|
||||
smtp_tls_policy_maps = lib.mkForce "";
|
||||
};
|
||||
|
||||
security.acme.acceptTerms = true;
|
||||
security.acme.defaults.email = "security@" + cfg.domain;
|
||||
|
||||
@@ -147,7 +147,7 @@ in {
|
||||
|
||||
(lib.mkIf cfg.watcher.enable {
|
||||
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
|
||||
sopsFile = "${flake}/sus/sentinella-default.yaml";
|
||||
sopsFile = flake + "/sus/sentinella-default.yaml";
|
||||
};
|
||||
|
||||
systemd.services."sentinella-watcher" = {
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
{ ... }: {
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
cfg = config.hectic.services.stable-video-diffusion;
|
||||
in {
|
||||
options.hectic.services.stable-video-diffusion = {
|
||||
enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
|
||||
|
||||
host = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
description = "Address the Stable Video Diffusion API binds to.";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 7861;
|
||||
description = "Port the Stable Video Diffusion API binds to.";
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.hectic.stable-video-diffusion-api;
|
||||
defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
|
||||
description = "Package providing the Stable Video Diffusion API executable.";
|
||||
};
|
||||
|
||||
modelId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "stabilityai/stable-video-diffusion-img2vid-xt";
|
||||
description = "Model identifier loaded by the Stable Video Diffusion API.";
|
||||
};
|
||||
|
||||
device = lib.mkOption {
|
||||
type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
|
||||
default = null;
|
||||
description = ''
|
||||
Torch device requested from the Stable Video Diffusion API. When null,
|
||||
the package keeps its own auto-detection behavior.
|
||||
'';
|
||||
};
|
||||
|
||||
libraryPath = lib.mkOption {
|
||||
type = with lib.types; listOf str;
|
||||
default = [];
|
||||
description = ''
|
||||
Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
|
||||
should include /run/opengl-driver/lib so libcuda.so is visible to torch.
|
||||
'';
|
||||
};
|
||||
|
||||
stateDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/stable-video-diffusion-api";
|
||||
description = "Persistent state directory for the Stable Video Diffusion API.";
|
||||
};
|
||||
|
||||
cacheDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/cache/stable-video-diffusion-api";
|
||||
description = "Cache directory for downloaded model and runtime artifacts.";
|
||||
};
|
||||
|
||||
outputDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "${cfg.stateDir}/outputs";
|
||||
defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
|
||||
description = "Directory where generated video outputs are written.";
|
||||
};
|
||||
|
||||
environmentFile = lib.mkOption {
|
||||
type = with lib.types; nullOr path;
|
||||
default = null;
|
||||
description = ''
|
||||
Optional environment file for secrets or runtime overrides. Values from
|
||||
the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
|
||||
SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
|
||||
and LD_LIBRARY_PATH by default.
|
||||
'';
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Whether to open the API port in the firewall.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.device != "cuda" || cfg.libraryPath != [];
|
||||
message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
|
||||
}
|
||||
];
|
||||
|
||||
users.users.stable-video-diffusion = {
|
||||
isSystemUser = true;
|
||||
group = "stable-video-diffusion";
|
||||
};
|
||||
users.groups.stable-video-diffusion = {};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||
"d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||
"d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||
];
|
||||
|
||||
systemd.services.stable-video-diffusion-api = {
|
||||
description = "Stable Video Diffusion HTTP API";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
environment = {
|
||||
SVD_API_HOST = cfg.host;
|
||||
SVD_API_PORT = toString cfg.port;
|
||||
SVD_MODEL_ID = cfg.modelId;
|
||||
SVD_STATE_DIR = cfg.stateDir;
|
||||
SVD_CACHE_DIR = cfg.cacheDir;
|
||||
SVD_OUTPUT_DIR = cfg.outputDir;
|
||||
}
|
||||
// lib.optionalAttrs (cfg.device != null) {
|
||||
SVD_DEVICE = cfg.device;
|
||||
}
|
||||
// lib.optionalAttrs (cfg.libraryPath != []) {
|
||||
LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
|
||||
};
|
||||
serviceConfig = lib.mkMerge [
|
||||
{
|
||||
Type = "simple";
|
||||
User = "stable-video-diffusion";
|
||||
Group = "stable-video-diffusion";
|
||||
WorkingDirectory = cfg.stateDir;
|
||||
ExecStart = lib.getExe' cfg.package "stable-video-diffusion-api";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
TimeoutStopSec = "30s";
|
||||
KillSignal = "SIGTERM";
|
||||
KillMode = "mixed";
|
||||
StandardOutput = "journal";
|
||||
StandardError = "journal";
|
||||
}
|
||||
(lib.mkIf (cfg.environmentFile != null) {
|
||||
EnvironmentFile = cfg.environmentFile;
|
||||
})
|
||||
];
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
|
||||
};
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
# Matrix Cluster Failover Runbook (`accord.tube`)
|
||||
|
||||
Primary: `hectic-lab` (NL, `128.140.75.58`)
|
||||
Standby: `bfs.poland.xray` (PL, `91.198.166.181`)
|
||||
|
||||
Module: `hectic.generic.matrix-cluster` (`nixos/module/generic/matrix-cluster.nix`).
|
||||
Shared secrets: `sus/matrix-cluster.yaml`.
|
||||
|
||||
All `psql` and `pg_ctl` invocations use PostgreSQL **17** at data dir
|
||||
`/var/lib/postgresql/17`.
|
||||
|
||||
## Initial setup
|
||||
|
||||
### 1. Provision shared SOPS file (`sus/matrix-cluster.yaml`)
|
||||
|
||||
On a workstation with both yukkop and yukkop-alt age keys available:
|
||||
|
||||
```sh
|
||||
sudo cat /var/lib/matrix-synapse/homeserver.signing.key # on NL (hectic-lab)
|
||||
# Copy the single line value into the buffer for the next step.
|
||||
|
||||
sops sus/matrix-cluster.yaml
|
||||
```
|
||||
|
||||
Populate the editor with:
|
||||
|
||||
```yaml
|
||||
matrix:
|
||||
signing-key: <paste verbatim signing-key line from NL>
|
||||
postgres-replication-password: <openssl rand -base64 32>
|
||||
object-storage:
|
||||
credentials: |
|
||||
ACCESS_KEY_ID=<verbatim copy from sus/hectic-lab.yaml>
|
||||
SECRET_ACCESS_KEY=<verbatim copy from sus/hectic-lab.yaml>
|
||||
porkbun-api-key: <PORKBUN_API_KEY>
|
||||
porkbun-secret-api-key: <PORKBUN_SECRET_API_KEY>
|
||||
```
|
||||
|
||||
Verify recipients:
|
||||
|
||||
```sh
|
||||
sops updatekeys sus/matrix-cluster.yaml
|
||||
sops -d sus/matrix-cluster.yaml | grep -E 'signing-key|porkbun-api-key|object-storage'
|
||||
```
|
||||
|
||||
Expected: all five keys present, exit 0.
|
||||
|
||||
### 2. Deploy NL primary first
|
||||
|
||||
```sh
|
||||
nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux' --target-host root@128.140.75.58
|
||||
```
|
||||
|
||||
Verify on NL:
|
||||
|
||||
```sh
|
||||
sudo systemctl status matrix-synapse postgresql matrix-cluster-replication-password
|
||||
sudo -u postgres psql -c "select rolname, rolreplication from pg_roles where rolname='replication';"
|
||||
# Expected: replication | t
|
||||
```
|
||||
|
||||
### 3. Seed PL replica with `pg_basebackup`
|
||||
|
||||
On PL:
|
||||
|
||||
```sh
|
||||
sudo systemctl stop postgresql
|
||||
sudo rm -rf /var/lib/postgresql/17
|
||||
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
|
||||
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
|
||||
pg_basebackup \
|
||||
-h 128.140.75.58 \
|
||||
-p 5432 \
|
||||
-U replication \
|
||||
-D /var/lib/postgresql/17 \
|
||||
-Fp -Xs -P -R \
|
||||
--no-password
|
||||
```
|
||||
|
||||
`-R` writes `standby.signal` and an initial `primary_conninfo`. The
|
||||
matrix-cluster module's `matrix-cluster-standby-bootstrap` service will
|
||||
overwrite `primary_conninfo` to use a libpq passfile on next boot.
|
||||
|
||||
### 4. Deploy PL standby
|
||||
|
||||
```sh
|
||||
nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux' --target-host root@91.198.166.181
|
||||
sudo systemctl start postgresql
|
||||
```
|
||||
|
||||
Verify streaming on NL:
|
||||
|
||||
```sh
|
||||
sudo -u postgres psql -c 'select client_addr, state, sync_state from pg_stat_replication;'
|
||||
# Expected: 91.198.166.181 | streaming | async
|
||||
```
|
||||
|
||||
Verify standby on PL:
|
||||
|
||||
```sh
|
||||
sudo -u postgres psql -c 'select pg_is_in_recovery();'
|
||||
# Expected: t
|
||||
sudo systemctl is-active matrix-synapse
|
||||
# Expected: inactive (standby keeps Synapse off)
|
||||
```
|
||||
|
||||
### 5. Remove duplicate S3 credentials from `sus/hectic-lab.yaml`
|
||||
|
||||
Only AFTER NL is confirmed healthy reading from the new shared file:
|
||||
|
||||
```sh
|
||||
sops sus/hectic-lab.yaml
|
||||
# Delete the matrix/object-storage/credentials block.
|
||||
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
|
||||
```
|
||||
|
||||
## Normal operations
|
||||
|
||||
```sh
|
||||
# NL: replication health
|
||||
sudo -u postgres psql -c 'select * from pg_stat_replication;'
|
||||
# Expected: 1 row, state=streaming, sync_state=async
|
||||
|
||||
# PL: replay status
|
||||
sudo -u postgres psql -c 'select now() - pg_last_xact_replay_timestamp() as lag;'
|
||||
|
||||
# Both: cert renewal
|
||||
sudo systemctl status acme-accord.tube.timer
|
||||
sudo journalctl -u acme-accord.tube.service --since '24 hours ago'
|
||||
|
||||
# Synapse health (NL primary)
|
||||
curl -sf https://accord.tube/_matrix/client/versions | head
|
||||
```
|
||||
|
||||
## Planned failover (NL -> PL)
|
||||
|
||||
```sh
|
||||
# 1. Drain NL: stop accepting writes.
|
||||
sudo systemctl stop matrix-synapse
|
||||
sudo systemctl stop postgresql # ensure no new WAL after this point
|
||||
|
||||
# 2. Promote PL replica.
|
||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
|
||||
# Wait until pg_is_in_recovery() returns f:
|
||||
sudo -u postgres psql -c 'select pg_is_in_recovery();'
|
||||
|
||||
# 3. Make the role switch declarative before rebuilding.
|
||||
# Edit the flake so rebuilds match the promoted database state:
|
||||
# - nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
|
||||
# hectic.generic.matrix-cluster.role = "primary";
|
||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = true;
|
||||
# hectic.generic.matrix-cluster.secretsFile = config.sops.secrets."matrix/secrets".path;
|
||||
# - nixos/system/hectic-lab/hectic-lab.nix:
|
||||
# hectic.generic.matrix-cluster.role = "standby";
|
||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = false;
|
||||
# hectic.generic.matrix-cluster.replication.peerHost = "91.198.166.181";
|
||||
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "128.140.75.58/32" ];
|
||||
# (You will also need a matrix/secrets entry on PL - copy from NL via SOPS.)
|
||||
sudo nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux'
|
||||
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
|
||||
sudo systemctl status matrix-synapse
|
||||
|
||||
# 4. Swap DNS A record at Porkbun:
|
||||
# accord.tube A 91.198.166.181 (was 128.140.75.58)
|
||||
# TTL: set to 300 in advance of any planned failover.
|
||||
# Porkbun UI: https://porkbun.com/account/domainsSpeedy -> accord.tube -> DNS -> edit A record.
|
||||
# Or via API:
|
||||
sudo curl -sX POST https://api.porkbun.com/api/json/v3/dns/editByNameType/accord.tube/A \
|
||||
-H 'content-type: application/json' \
|
||||
-d "$(jq -n --arg k "$PORKBUN_API_KEY" --arg s "$PORKBUN_SECRET_API_KEY" \
|
||||
'{secretapikey:$s,apikey:$k,content:"91.198.166.181",ttl:"300"}')"
|
||||
|
||||
# 5. Federation smoke test.
|
||||
curl -s 'https://federationtester.matrix.org/api/report?server_name=accord.tube' | jq .FederationOK
|
||||
# Expected: true
|
||||
```
|
||||
|
||||
Expected after the rebuilds:
|
||||
|
||||
- `bfs.poland.xray` evaluates and runs as `role = "primary"`.
|
||||
- `hectic-lab` evaluates as `role = "standby"` with Synapse forced off.
|
||||
- Future `nixos-rebuild` runs preserve the promoted topology instead of reapplying standby settings to PL.
|
||||
|
||||
## Failback (PL -> NL)
|
||||
|
||||
```sh
|
||||
# 1. Stop NL postgres if still up; clear its data dir.
|
||||
sudo systemctl stop postgresql matrix-synapse
|
||||
sudo rm -rf /var/lib/postgresql/17
|
||||
|
||||
# 2. Re-seed NL from PL (now the live primary).
|
||||
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
|
||||
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
|
||||
pg_basebackup -h 91.198.166.181 -p 5432 -U replication \
|
||||
-D /var/lib/postgresql/17 -Fp -Xs -P -R --no-password
|
||||
|
||||
# 3. Temporarily flip roles in the flake:
|
||||
# - hectic-lab.nix: role = "standby"; peerHost = "91.198.166.181";
|
||||
# - bfs.poland.xray.nix: role = "primary"; peerHost = "128.140.75.58";
|
||||
# Rebuild both.
|
||||
|
||||
# 4. Once NL is streaming green, do the reverse failover dance:
|
||||
sudo systemctl stop matrix-synapse # on PL
|
||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote # on NL
|
||||
# Then revert the flake role assignments back to NL=primary / PL=standby and
|
||||
# rebuild both hosts.
|
||||
|
||||
# 5. Swap DNS back at Porkbun (A -> 128.140.75.58).
|
||||
```
|
||||
|
||||
## Disaster recovery (NL permanently lost)
|
||||
|
||||
```sh
|
||||
# 1. Promote PL as the new permanent primary.
|
||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
|
||||
|
||||
# 2. Edit nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
|
||||
# hectic.generic.matrix-cluster.role = "primary";
|
||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = lib.mkForce null;
|
||||
# hectic.generic.matrix-cluster.replication.peerHost = "<new-standby-ip>";
|
||||
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "<new-standby-ip>/32" ];
|
||||
|
||||
# 3. Provision a new host (any region with Porkbun-managed DNS) and import
|
||||
# self.nixosModules.matrix-cluster with role = "standby" pointed at PL's IP.
|
||||
|
||||
# 4. Bootstrap the new standby via pg_basebackup from PL exactly as in
|
||||
# "Initial setup" step 3, replacing 128.140.75.58 with PL's IP.
|
||||
|
||||
# 5. Update Porkbun A record to PL's IP permanently.
|
||||
```
|
||||
@@ -9,16 +9,11 @@
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
matrixBackend = "https://128.140.75.58";
|
||||
matrixHost = "accord.tube";
|
||||
jitsiHost = "meet.accord.tube";
|
||||
elementEntryDomain = "element.bfs.band";
|
||||
polandEntryDomain = "bfs.band";
|
||||
backendProxyConfig = ''
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name ${matrixHost};
|
||||
proxy_set_header Host ${matrixHost};
|
||||
'';
|
||||
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||
in {
|
||||
imports = [
|
||||
self.nixosModules.xray-system
|
||||
@@ -46,7 +41,6 @@ in {
|
||||
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
|
||||
};
|
||||
replication = {
|
||||
peerHost = "128.140.75.58";
|
||||
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
|
||||
};
|
||||
acme = {
|
||||
@@ -71,11 +65,31 @@ in {
|
||||
hostName = jitsiHost;
|
||||
};
|
||||
|
||||
# NOTE(yukkop): disk was provisioned outside disko, so the expected partition
|
||||
# label does not exist. Pin root to the live filesystem UUID so stage 1 can
|
||||
# mount `/` reliably.
|
||||
fileSystems."/" = lib.mkForce {
|
||||
device = "/dev/disk/by-uuid/06b48ef1-a1eb-428d-821c-90c96a624542";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
security.acme = {
|
||||
acceptTerms = true;
|
||||
defaults.email = "security@bfs.band";
|
||||
};
|
||||
|
||||
# NOTE(yukkop): this host gets an IPv6 route via RA, but object storage
|
||||
# fetches to hel1.your-objectstorage.com currently stall over IPv6 while
|
||||
# IPv4 works. Synapse's S3 media backend uses getaddrinfo ordering, so
|
||||
# prefer IPv4 here to keep Element media downloads responsive.
|
||||
environment.etc."gai.conf".text = ''
|
||||
precedence ::ffff:0:0/96 100
|
||||
'';
|
||||
|
||||
systemd.services.matrix-synapse.restartTriggers = [
|
||||
config.environment.etc."gai.conf".source
|
||||
];
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
|
||||
@@ -112,19 +126,17 @@ in {
|
||||
};
|
||||
|
||||
locations."= /livekit/jwt" = {
|
||||
proxyPass = "${matrixBackend}/livekit/jwt";
|
||||
extraConfig = backendProxyConfig;
|
||||
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
|
||||
};
|
||||
|
||||
locations."^~ /livekit/jwt/" = {
|
||||
proxyPass = "${matrixBackend}/livekit/jwt/";
|
||||
extraConfig = backendProxyConfig;
|
||||
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
|
||||
};
|
||||
|
||||
locations."= /livekit/sfu" = {
|
||||
proxyPass = "${matrixBackend}/livekit/sfu";
|
||||
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = backendProxyConfig + ''
|
||||
extraConfig = ''
|
||||
proxy_send_timeout 120;
|
||||
proxy_read_timeout 120;
|
||||
proxy_buffering off;
|
||||
@@ -135,9 +147,9 @@ in {
|
||||
};
|
||||
|
||||
locations."^~ /livekit/sfu/" = {
|
||||
proxyPass = "${matrixBackend}/livekit/sfu/";
|
||||
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = backendProxyConfig + ''
|
||||
extraConfig = ''
|
||||
proxy_send_timeout 120;
|
||||
proxy_read_timeout 120;
|
||||
proxy_buffering off;
|
||||
@@ -148,13 +160,14 @@ in {
|
||||
};
|
||||
|
||||
locations."^~ /_matrix/" = {
|
||||
proxyPass = "${matrixBackend}/_matrix/";
|
||||
extraConfig = backendProxyConfig;
|
||||
proxyPass = "http://127.0.0.1:8008";
|
||||
extraConfig = ''
|
||||
client_max_body_size ${config.hectic.generic.matrix-cluster.maxUploadSize};
|
||||
'';
|
||||
};
|
||||
|
||||
locations."^~ /_synapse/client/" = {
|
||||
proxyPass = "${matrixBackend}/_synapse/client/";
|
||||
extraConfig = backendProxyConfig;
|
||||
proxyPass = "http://127.0.0.1:8008";
|
||||
};
|
||||
};
|
||||
|
||||
@@ -164,7 +177,7 @@ in {
|
||||
|
||||
locations."= /config.${elementEntryDomain}.json".return = "302 /config.json";
|
||||
|
||||
root = pkgs.element-web.override {
|
||||
root = pkgs.hectic.element-web.override {
|
||||
conf = {
|
||||
default_server_config = {
|
||||
"m.homeserver".base_url = "https://${polandEntryDomain}";
|
||||
@@ -176,6 +189,8 @@ in {
|
||||
preferred_domain = jitsiHost;
|
||||
};
|
||||
|
||||
hectic.videoMessages.enabled = true;
|
||||
|
||||
room_directory.servers = [ matrixHost ];
|
||||
|
||||
default_theme = "dark";
|
||||
@@ -189,41 +204,41 @@ in {
|
||||
key = "matrix/signing-key";
|
||||
owner = "matrix-synapse";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
sops.secrets."matrix/postgres-replication-password" = {
|
||||
key = "matrix/postgres-replication-password";
|
||||
owner = "postgres";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
sops.secrets."matrix/object-storage/credentials" = {
|
||||
key = "matrix/object-storage/credentials";
|
||||
owner = "matrix-synapse";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
sops.secrets."matrix/secrets" = {
|
||||
key = "matrix/secrets";
|
||||
owner = "matrix-synapse";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
sops.secrets."matrix/turn-secret" = {
|
||||
key = "matrix/turn-secret";
|
||||
owner = "turnserver";
|
||||
group = "turnserver";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
sops.secrets."matrix/porkbun-api-key" = {
|
||||
key = "matrix/porkbun-api-key";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
sops.secrets."matrix/porkbun-secret-api-key" = {
|
||||
key = "matrix/porkbun-secret-api-key";
|
||||
mode = "0400";
|
||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
||||
sopsFile = matrixClusterSopsFile;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
domain,
|
||||
...
|
||||
}: {
|
||||
config,
|
||||
...
|
||||
}: {
|
||||
hectic.services.attic = {
|
||||
enable = true;
|
||||
hostName = "cache.${domain}";
|
||||
port = 8081;
|
||||
environmentFile = config.sops.secrets."atticd/environment".path;
|
||||
storage = {
|
||||
bucket = "cache-hectic-lab";
|
||||
endpoint = "https://hel1.your-objectstorage.com";
|
||||
region = "hel1";
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx.virtualHosts."cache.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
extraConfig = ''
|
||||
client_max_body_size 0;
|
||||
'';
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:8081";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
domain,
|
||||
...
|
||||
}: {
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
enteDomain = "ente.${domain}";
|
||||
in {
|
||||
hectic.services.ente = {
|
||||
enable = true;
|
||||
apiDomain = "api.${enteDomain}";
|
||||
disableRegistration = false;
|
||||
|
||||
domains = {
|
||||
accounts = "accounts.${enteDomain}";
|
||||
cast = "cast.${enteDomain}";
|
||||
albums = "albums.${enteDomain}";
|
||||
photos = "photos.${enteDomain}";
|
||||
};
|
||||
|
||||
smtp = {
|
||||
enable = true;
|
||||
host = "mail.${domain}";
|
||||
email = "security@${domain}";
|
||||
};
|
||||
|
||||
storage = {
|
||||
bucket = "ente-hectic-lab";
|
||||
endpoint = "https://hel1.your-objectstorage.com";
|
||||
region = "hel1";
|
||||
};
|
||||
|
||||
secrets = {
|
||||
encryptionKeyFile = config.sops.secrets."ente/key-encryption".path;
|
||||
hashKeyFile = config.sops.secrets."ente/key-hash".path;
|
||||
jwtSecretFile = config.sops.secrets."ente/jwt-secret".path;
|
||||
s3AccessKeyFile = config.sops.secrets."ente/s3-access-key".path;
|
||||
s3SecretKeyFile = config.sops.secrets."ente/s3-secret-key".path;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
port = 22222;
|
||||
stateDir = "/var/lib/experimental-sshd";
|
||||
configFile = pkgs.writeText "experimental-sshd_config" ''
|
||||
Port 22222
|
||||
ListenAddress 0.0.0.0
|
||||
ListenAddress ::
|
||||
|
||||
HostKey ${stateDir}/ssh_host_ed25519_key
|
||||
HostKey ${stateDir}/ssh_host_rsa_key
|
||||
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
PubkeyAuthentication yes
|
||||
PermitRootLogin prohibit-password
|
||||
UsePAM yes
|
||||
AuthenticationMethods publickey
|
||||
AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u
|
||||
LogLevel DEBUG3
|
||||
|
||||
VersionAddendum none
|
||||
HostKeyAlgorithms rsa-sha2-512,rsa-sha2-256,ssh-ed25519
|
||||
KexAlgorithms curve25519-sha256,diffie-hellman-group14-sha256
|
||||
Ciphers aes256-ctr,aes128-ctr
|
||||
MACs hmac-sha2-512,hmac-sha2-256
|
||||
'';
|
||||
|
||||
keygenScript = pkgs.writeShellScript "experimental-sshd-keygen" ''
|
||||
set -eu
|
||||
|
||||
${pkgs.coreutils}/bin/mkdir -p -m 0700 ${stateDir}
|
||||
|
||||
if [ ! -f ${stateDir}/ssh_host_ed25519_key ]; then
|
||||
${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f ${stateDir}/ssh_host_ed25519_key -N ""
|
||||
fi
|
||||
|
||||
if [ ! -f ${stateDir}/ssh_host_rsa_key ]; then
|
||||
${pkgs.openssh}/bin/ssh-keygen -t rsa -b 4096 -f ${stateDir}/ssh_host_rsa_key -N ""
|
||||
fi
|
||||
'';
|
||||
in
|
||||
{
|
||||
environment.etc."ssh/experimental-sshd_config".source = configFile;
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ port ];
|
||||
|
||||
systemd.services.experimental-sshd-keygen = {
|
||||
description = "Generate experimental SSH host keys";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
StateDirectory = "experimental-sshd";
|
||||
ExecStart = keygenScript;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.experimental-sshd = {
|
||||
description = "Experimental SSH daemon";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "experimental-sshd-keygen.service" ];
|
||||
after = [ "network.target" "experimental-sshd-keygen.service" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
StateDirectory = "experimental-sshd";
|
||||
RuntimeDirectory = "experimental-sshd";
|
||||
ExecStart = "${pkgs.openssh}/bin/sshd -D -e -f /etc/ssh/experimental-sshd_config";
|
||||
};
|
||||
preStart = ''
|
||||
${pkgs.openssh}/bin/sshd -t -f /etc/ssh/experimental-sshd_config
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -15,7 +15,7 @@ with builtins;
|
||||
with lib;
|
||||
let
|
||||
domain = "hectic-lab.com";
|
||||
matrixDomain = "accord.tube";
|
||||
sshPort = 22;
|
||||
mailUserNames = [
|
||||
"security"
|
||||
"founders"
|
||||
@@ -25,6 +25,7 @@ let
|
||||
"iana-perlyk"
|
||||
"snuff"
|
||||
"antoshka"
|
||||
"evgenii-kazakov"
|
||||
];
|
||||
mkMailPasswordSecret = name: {
|
||||
name = "mailserver/${name}/hashedPassword";
|
||||
@@ -36,9 +37,12 @@ let
|
||||
hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path;
|
||||
};
|
||||
};
|
||||
sslOpts = {
|
||||
sslCertificate = config.sops.secrets."ssl/porkbun/${domain}/domain.cert.pem".path;
|
||||
sslCertificateKey = config.sops.secrets."ssl/porkbun/${domain}/private.key.pem".path;
|
||||
mkEnteSecret = name: {
|
||||
name = "ente/${name}";
|
||||
value = {
|
||||
owner = "ente";
|
||||
group = "ente";
|
||||
};
|
||||
};
|
||||
in {
|
||||
imports = [
|
||||
@@ -51,9 +55,12 @@ in {
|
||||
|
||||
inputs.hectic-landing.nixosModules.hectic-landing
|
||||
|
||||
(import ./attic.nix { inherit flake self inputs domain; })
|
||||
(import ./containers.nix { inherit flake self inputs; })
|
||||
(import ./mechabellum.nix { inherit flake self inputs domain sslOpts; })
|
||||
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain sslOpts; })
|
||||
./experimental-sshd.nix
|
||||
(import ./ente.nix { inherit domain; })
|
||||
(import ./mechabellum.nix { inherit flake self inputs domain; })
|
||||
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
|
||||
];
|
||||
|
||||
services.hectic-landing = {
|
||||
@@ -74,6 +81,7 @@ in {
|
||||
enable = true;
|
||||
networkMatchConfigName = "enp1s0";
|
||||
ipv4 = "128.140.75.58";
|
||||
floatingIpv4 = "78.47.243.0";
|
||||
ipv6 = "2a01:4f8:c2c:d54a";
|
||||
};
|
||||
services.matrix = {
|
||||
@@ -100,6 +108,7 @@ in {
|
||||
'';
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
git
|
||||
rsync
|
||||
python311
|
||||
@@ -110,28 +119,25 @@ in {
|
||||
sops = {
|
||||
gnupg.sshKeyPaths = [ ];
|
||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||
defaultSopsFile = "${flake}/sus/hectic-lab.yaml";
|
||||
defaultSopsFile = flake + "/sus/hectic-lab.yaml";
|
||||
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
|
||||
"init-postgresql" = {
|
||||
key = "init-postgresql";
|
||||
};
|
||||
"ssl/porkbun/${domain}/domain.cert.pem" = {
|
||||
group = "nginx";
|
||||
mode = "0440";
|
||||
};
|
||||
"ssl/porkbun/${domain}/private.key.pem" = {
|
||||
group = "nginx";
|
||||
mode = "0440";
|
||||
};
|
||||
"ssl/porkbun/${domain}/public.key.pem" = {
|
||||
group = "nginx";
|
||||
mode = "0440";
|
||||
};
|
||||
"atticd/environment" = {};
|
||||
"wg-bfs/private-key" = {};
|
||||
};
|
||||
} // builtins.listToAttrs (map mkEnteSecret [
|
||||
"key-encryption"
|
||||
"key-hash"
|
||||
"jwt-secret"
|
||||
"s3-access-key"
|
||||
"s3-secret-key"
|
||||
]);
|
||||
};
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
# neuro machine
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
|
||||
# yukkop
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
|
||||
@@ -158,6 +164,8 @@ in {
|
||||
];
|
||||
};
|
||||
|
||||
services.openssh.ports = [ sshPort ];
|
||||
|
||||
services.mailserver = {
|
||||
enable = true;
|
||||
domain = domain;
|
||||
@@ -178,10 +186,10 @@ in {
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
sshPort # ssh
|
||||
80
|
||||
443
|
||||
3306 # mysql
|
||||
11012 # gitea ssh
|
||||
25565
|
||||
55228 # ss-bfs
|
||||
];
|
||||
@@ -204,8 +212,8 @@ in {
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
# NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module
|
||||
# (ACME-managed). See services.hectic-landing above.
|
||||
virtualHosts."store.${domain}" = sslOpts // {
|
||||
virtualHosts."store.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
root = "/var/www/store";
|
||||
locations."/" = {
|
||||
@@ -214,7 +222,8 @@ in {
|
||||
'';
|
||||
};
|
||||
};
|
||||
virtualHosts."snuff.${domain}" = sslOpts // {
|
||||
virtualHosts."snuff.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
extraConfig = ''
|
||||
@@ -223,7 +232,8 @@ in {
|
||||
'';
|
||||
};
|
||||
};
|
||||
virtualHosts."nrv.${domain}" = sslOpts // {
|
||||
virtualHosts."nrv.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
extraConfig = ''
|
||||
@@ -232,7 +242,8 @@ in {
|
||||
'';
|
||||
};
|
||||
};
|
||||
virtualHosts."yukkop.${domain}" = sslOpts // {
|
||||
virtualHosts."yukkop.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
extraConfig = ''
|
||||
@@ -257,10 +268,10 @@ in {
|
||||
gitea = {
|
||||
enable = true;
|
||||
package = pkgs.hectic.gitea-heatmap;
|
||||
settings.service.DISABLE_REGISTRATION = false;
|
||||
settings.service.DISABLE_REGISTRATION = true;
|
||||
settings.server = {
|
||||
HTTP_PORT = 11011;
|
||||
#SSH_PORT = 22;
|
||||
SSH_PORT = sshPort;
|
||||
SSH_DOMAIN = "hectic-lab.com";
|
||||
};
|
||||
database = {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
{
|
||||
inputs,
|
||||
domain,
|
||||
sslOpts,
|
||||
...
|
||||
}: {
|
||||
pkgs,
|
||||
@@ -29,7 +28,8 @@ in {
|
||||
enable = true;
|
||||
};
|
||||
|
||||
services.nginx.virtualHosts."${mechDomain}" = sslOpts // {
|
||||
services.nginx.virtualHosts."${mechDomain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
root = inputs.mechabellum-replay-analysis.packages.${system}.frontend;
|
||||
|
||||
@@ -37,6 +37,7 @@ in {
|
||||
proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
|
||||
extraConfig = ''
|
||||
proxy_http_version 1.1;
|
||||
client_max_body_size 500M;
|
||||
'';
|
||||
};
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
flake,
|
||||
self,
|
||||
domain,
|
||||
sslOpts,
|
||||
...
|
||||
}: { ... }: {
|
||||
hectic.services."sentinèlla" = {
|
||||
@@ -17,7 +16,8 @@
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
virtualHosts."probe.${domain}" = sslOpts // {
|
||||
virtualHosts."probe.${domain}" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:5988";
|
||||
|
||||
@@ -15,32 +15,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
|
||||
self.overlays.default
|
||||
inputs.nix-minecraft.overlay
|
||||
];
|
||||
config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
||||
config.allowUnfreePredicate = pkg:
|
||||
self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
||||
"minecraft-server"
|
||||
"neoforge"
|
||||
|
||||
"nvidia-x11"
|
||||
|
||||
"cuda_nvcc"
|
||||
"cuda_cudart"
|
||||
"cuda_cuobjdump"
|
||||
"cuda_cupti"
|
||||
"cuda_nvdisasm"
|
||||
"cuda_cccl"
|
||||
"cuda_nvml_dev"
|
||||
"cuda_nvrtc"
|
||||
"cuda_nvtx"
|
||||
"cuda_profiler_api"
|
||||
|
||||
"libcusparse_lt"
|
||||
"libcublas"
|
||||
"libcufft"
|
||||
"libcufile"
|
||||
"libcurand"
|
||||
"libcusolver"
|
||||
"libnvjitlink"
|
||||
"libcusparse"
|
||||
"cudnn"
|
||||
];
|
||||
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
|
||||
config.permittedInsecurePackages = [
|
||||
|
||||
@@ -17,6 +17,11 @@
|
||||
|
||||
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
|
||||
|
||||
stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
|
||||
pkgs.stdenv.cc.cc.lib
|
||||
pkgs.zlib
|
||||
];
|
||||
|
||||
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
|
||||
pname = "ollama";
|
||||
version = "0.22.1";
|
||||
@@ -176,6 +181,19 @@ in {
|
||||
openFirewall = false;
|
||||
};
|
||||
|
||||
hectic.services.stable-video-diffusion = {
|
||||
enable = true;
|
||||
host = "127.0.0.1";
|
||||
port = 7861;
|
||||
package = pkgs.hectic.stable-video-diffusion-api;
|
||||
device = "cuda";
|
||||
libraryPath = [
|
||||
stableVideoDiffusionLibraryPath
|
||||
"/run/opengl-driver/lib"
|
||||
];
|
||||
openFirewall = false;
|
||||
};
|
||||
|
||||
networking = {
|
||||
networkmanager.enable = true;
|
||||
useDHCP = lib.mkDefault true;
|
||||
@@ -263,6 +281,7 @@ in {
|
||||
in [
|
||||
#python-ai
|
||||
git
|
||||
hectic.hiddify-core
|
||||
neovim
|
||||
wget
|
||||
ethtool
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
flake,
|
||||
self,
|
||||
inputs,
|
||||
system ? "x86_64-linux",
|
||||
...
|
||||
}: let
|
||||
# Use folder name as name of this system
|
||||
name = builtins.baseNameOf ./.;
|
||||
|
||||
in self.lib.nixpkgs-lib.nixosSystem {
|
||||
pkgs = import inputs.nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ self.overlays.default ];
|
||||
};
|
||||
modules = [
|
||||
{ networking.hostName = name; }
|
||||
(import ./${name}.nix { inherit flake self inputs; })
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
inputs,
|
||||
self,
|
||||
...
|
||||
}: {
|
||||
pkgs,
|
||||
modulesPath,
|
||||
...
|
||||
}: {
|
||||
imports = [
|
||||
(modulesPath + "/profiles/qemu-guest.nix")
|
||||
inputs.sops-nix.nixosModules.sops
|
||||
self.nixosModules.hectic
|
||||
];
|
||||
|
||||
hectic = {
|
||||
archetype.dev.enable = true;
|
||||
hardware.hetzner-cloud = {
|
||||
enable = true;
|
||||
device = "/dev/sda";
|
||||
networkMatchConfigMac = "92:00:08:4a:b0:32";
|
||||
ipv4 = "46.225.237.218";
|
||||
ipv6 = "2a01:4f8:c2c:3b14";
|
||||
};
|
||||
};
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
|
||||
];
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
git
|
||||
rsync
|
||||
];
|
||||
}
|
||||
+37
-11
@@ -1,18 +1,21 @@
|
||||
# db-tool
|
||||
|
||||
PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries.
|
||||
PostgreSQL utility package exposing separate development and operations binaries.
|
||||
`db-dev` preserves the current development workflow via the `database` binary;
|
||||
`db-ops` provides target-safe operational helpers such as secrets hydration.
|
||||
|
||||
## Provided Binaries
|
||||
|
||||
| Binary | Description |
|
||||
| --- | --- |
|
||||
| `database` | Main script for managing migrations, deployments, and logs. |
|
||||
| `database` | Main `db-dev` script for managing local development databases, migrations, deployments, and logs. |
|
||||
| `db-ops` | Operational helper binary for target/runtime database actions. |
|
||||
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
|
||||
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
|
||||
|
||||
## Required Environment Variables
|
||||
|
||||
These variables must be set for `db-tool` to function.
|
||||
These variables must be set for `db-dev` / `database` to function.
|
||||
|
||||
| Variable | Description |
|
||||
| --- | --- |
|
||||
@@ -39,7 +42,7 @@ These variables must be set for `db-tool` to function.
|
||||
|
||||
## Postgres Package Override
|
||||
|
||||
By default, `db-tool`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
|
||||
By default, `db-dev`/`db-ops`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
|
||||
|
||||
```nix
|
||||
let
|
||||
@@ -48,7 +51,8 @@ let
|
||||
]);
|
||||
in {
|
||||
packages = [
|
||||
(pkgs.hectic."db-tool".override { postgresql = myPg; })
|
||||
(pkgs.hectic."db-dev".override { postgresql = myPg; })
|
||||
(pkgs.hectic."db-ops".override { postgresql = myPg; })
|
||||
(pkgs.hectic."postgres-init".override { postgresql = myPg; })
|
||||
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
|
||||
];
|
||||
@@ -66,6 +70,28 @@ The `pull_staging` subcommand allows importing data from a remote staging enviro
|
||||
|
||||
If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr.
|
||||
|
||||
## normalize-backup Contract
|
||||
|
||||
The `normalize-backup` subcommand converts a physical PostgreSQL backup into a
|
||||
local-development restore artifact without modifying the input backup:
|
||||
|
||||
```sh
|
||||
database normalize-backup [OPTIONS] [path]
|
||||
```
|
||||
|
||||
The input `path` defaults to `${LOCAL_DIR}/focus/postgresql-backup` and must be a
|
||||
backup directory compatible with `database restore`, containing `base.tar.gz`
|
||||
and optionally `pg_wal.tar.gz`. The output defaults to a normalized backup path
|
||||
and can be overridden with `--output <path>`. The output directory must not be
|
||||
the same path as the input and must not be inside the input directory.
|
||||
|
||||
Normalization extracts the physical backup into temporary PGDATA, replaces
|
||||
production PostgreSQL access/configuration with local restore-safe
|
||||
`postgresql.conf` and `pg_hba.conf` files, removes standby/recovery/runtime
|
||||
leftovers, starts the cluster locally, ensures the requested `--role` and
|
||||
`--database` exist, stops cleanly, and repacks a backup directory that can be
|
||||
used by `database restore`.
|
||||
|
||||
## Subcommands
|
||||
|
||||
- `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success.
|
||||
@@ -73,20 +99,21 @@ If any of these variables are missing when `pull_staging` is invoked, the tool w
|
||||
- `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`.
|
||||
- `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster.
|
||||
- `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`.
|
||||
- `normalize-backup`: Rewrite a physical backup artifact for local restore use.
|
||||
- `pull_staging`: Import data from the staging environment based on the env contract.
|
||||
- `init`: Wrapper around `postgres-init` to start the cluster.
|
||||
- `migrator`: Directly invoke the migration tool with the correct environment context.
|
||||
|
||||
## shellHook Example
|
||||
|
||||
To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
|
||||
To use `db-dev` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
|
||||
|
||||
```nix
|
||||
{
|
||||
# ...
|
||||
devShells.default = pkgs.mkShell {
|
||||
packages = [
|
||||
pkgs.hectic.db-tool
|
||||
pkgs.hectic.db-dev
|
||||
pkgs.hectic.postgres-init
|
||||
pkgs.hectic.postgres-cleanup
|
||||
];
|
||||
@@ -111,7 +138,7 @@ To use `db-tool` in a Nix development shell, add the following to your `flake.ni
|
||||
|
||||
## hectic Bundle
|
||||
|
||||
`db-tool` and `migrator` apply a single bundle of SQL files that bootstrap the
|
||||
`db-dev`, `db-ops`, and `migrator` apply a single bundle of SQL files that bootstrap the
|
||||
`hectic` schema. The bundle lives in
|
||||
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
|
||||
contract, file layout, and the `self.lib.hectic.*` Nix API.
|
||||
@@ -167,6 +194,7 @@ ALTER DATABASE mydb SET hectic.inheritance_extra_excluded_schemas = 'legacy,etl'
|
||||
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
|
||||
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
|
||||
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
|
||||
| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. |
|
||||
|
||||
The bundle is idempotent — repeated application is safe.
|
||||
|
||||
@@ -187,15 +215,13 @@ directly against the paths exposed by `self.lib.hectic.*.path`:
|
||||
|
||||
```nix
|
||||
services.postgresql.initialScript = pkgs.writeText "hectic-init.sql" ''
|
||||
\i ${self.lib.hectic.version.path}
|
||||
\i ${self.lib.hectic.secret.path}
|
||||
\i ${self.lib.hectic.migration.path}
|
||||
\i ${self.lib.hectic.inheritance.path}
|
||||
'';
|
||||
```
|
||||
|
||||
The version file (`self.lib.hectic.version`) is templated and only exposes
|
||||
`.sql` (a string). Materialize it with `pkgs.writeText` if a path is needed.
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Meaning |
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
: "${REMAINING_ARGS:=}"
|
||||
|
||||
: "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}"
|
||||
: "${DEFAULT_NORMALIZED_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup-normalized}"
|
||||
|
||||
: "${SCRIPT_NAME:=$(basename "$0")}"
|
||||
SCRIPT_NAME=${SCRIPT_NAME%%.sh}
|
||||
@@ -186,6 +187,9 @@ ${BGREEN}Database Subcommands:${NC}
|
||||
|
||||
${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup
|
||||
|
||||
${BCYAN}normalize-backup${NC} ${CYAN}[OPTIONS] [PATH]$NC
|
||||
Normalize a raw physical backup for local restore/diff
|
||||
|
||||
${BCYAN}backup${NC} Create database backup
|
||||
Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC
|
||||
|
||||
@@ -246,6 +250,7 @@ ${BGREEN}Examples:${NC}
|
||||
$SCRIPT_NAME init Initialize PostgreSQL only
|
||||
$SCRIPT_NAME restore Restore from default backup
|
||||
$SCRIPT_NAME restore /path/to/backup Restore from specific path
|
||||
$SCRIPT_NAME normalize-backup /path/to/raw-backup
|
||||
$SCRIPT_NAME backup Create backup
|
||||
$SCRIPT_NAME log Show database logs
|
||||
$SCRIPT_NAME log list List available log files
|
||||
@@ -421,6 +426,43 @@ EOF
|
||||
)" | "$PAGER_OR_CAT"
|
||||
}
|
||||
|
||||
help_normalize_backup() {
|
||||
# shellcheck disable=SC2059
|
||||
printf "$(cat <<EOF
|
||||
${BGREEN}Usage:${NC} $SCRIPT_NAME normalize-backup [OPTIONS] [path]
|
||||
|
||||
Normalize a raw PostgreSQL physical backup for local development.
|
||||
|
||||
This command leaves the input backup untouched. It extracts $BBLACK\`base.tar.gz\`$NC
|
||||
and optional $BBLACK\`pg_wal.tar.gz\`$NC into temporary PGDATA, writes local
|
||||
$BBLACK\`postgresql.conf\`$NC and $BBLACK\`pg_hba.conf\`$NC files, removes standby,
|
||||
recovery, and runtime leftovers, starts the cluster locally, ensures a login role
|
||||
and database exist, stops cleanly, then repacks a normalized backup directory.
|
||||
|
||||
${BGREEN}Arguments:${NC}
|
||||
${BCYAN}path$NC Input backup directory (optional)
|
||||
Defaults to $BBLACK$DEFAULT_BACKUP_PATH$NC
|
||||
|
||||
${BGREEN}Options:${NC}
|
||||
$BCYAN-o$NC, $BCYAN--output$NC ${CYAN}<path>$NC Output backup directory
|
||||
Defaults to $BBLACK$DEFAULT_NORMALIZED_BACKUP_PATH$NC
|
||||
$BCYAN--role$NC ${CYAN}<name>$NC Login role to ensure (default $BBLACK\$(id -un)$NC)
|
||||
$BCYAN--database$NC ${CYAN}<name>$NC Database to ensure (default $BBLACK\${PG_DATABASE:-testdb}$NC)
|
||||
$BCYAN--admin-role$NC ${CYAN}<name>$NC Role used for local maintenance
|
||||
Defaults to $BBLACK\${URI_USER:-postgres}$NC
|
||||
$BCYAN-h$NC, $BCYAN--help$NC Show this help message
|
||||
|
||||
${BGREEN}Files Created:${NC}
|
||||
- ${BBLACK}\`base.tar.gz\`$NC: Normalized database cluster backup
|
||||
|
||||
${BGREEN}Examples:${NC}
|
||||
$SCRIPT_NAME normalize-backup /path/to/raw-backup
|
||||
$SCRIPT_NAME normalize-backup --output focus/postgresql-backup-normalized
|
||||
|
||||
EOF
|
||||
)" | "$PAGER_OR_CAT"
|
||||
}
|
||||
|
||||
help_diff() {
|
||||
# shellcheck disable=SC2059
|
||||
printf "$(cat <<EOF
|
||||
@@ -441,6 +483,8 @@ ${BGREEN}Arguments:
|
||||
${BGREEN}Options:${NC}
|
||||
$BCYAN--tables${NC} ${CYAN}<list>${NC} Comma-separated list of tables to diff
|
||||
Example: --tables users,orders,products
|
||||
$BCYAN--ignore-migration-fail${NC}
|
||||
Continue diff even if DB1 migrations fail
|
||||
$BCYAN-m${NC}, $BCYAN--mock${NC} Mock external API calls when hydrating DB2
|
||||
Replaces HTTP-calling functions with stubs/test data
|
||||
$BCYAN-h${NC}, $BCYAN--help${NC} Show this help message
|
||||
@@ -464,6 +508,7 @@ ${BGREEN}Examples:${NC}
|
||||
$SCRIPT_NAME diff Compare using default backup
|
||||
$SCRIPT_NAME diff /path/to/backup Compare using specific backup
|
||||
$SCRIPT_NAME diff --tables users,orders Compare specific tables
|
||||
$SCRIPT_NAME diff --ignore-migration-fail Continue despite DB1 migration failure
|
||||
$SCRIPT_NAME diff -m Compare with external APIs mocked
|
||||
$SCRIPT_NAME diff log Show logs from diff operation
|
||||
|
||||
@@ -805,6 +850,11 @@ subcommand_restore() {
|
||||
RESTORE_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
|
||||
fi
|
||||
|
||||
local restore_database="${PG_DATABASE:-}"
|
||||
if [ -f "${RESTORE_BACKUP_PATH:?}/database_name" ]; then
|
||||
restore_database=$(tr -d '\n' < "${RESTORE_BACKUP_PATH:?}/database_name")
|
||||
fi
|
||||
|
||||
postgres-cleanup
|
||||
|
||||
local data="${PG_WORKING_DIR:?}/data"
|
||||
@@ -817,12 +867,243 @@ subcommand_restore() {
|
||||
tar -xzf "${RESTORE_BACKUP_PATH:?}/pg_wal.tar.gz" -C "${data}/pg_wal"
|
||||
fi
|
||||
|
||||
env PG_REUSE= postgres-init
|
||||
env PG_REUSE= PG_DATABASE="$restore_database" postgres-init
|
||||
|
||||
rm -f "${data}/standby.signal" "${data}/recovery.signal"
|
||||
restore_namespace
|
||||
}
|
||||
|
||||
___sql_literal() {
|
||||
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/''/g")"
|
||||
}
|
||||
|
||||
___normalize_backup_remove_leftovers() {
|
||||
local pgdata="$1"
|
||||
rm -f \
|
||||
"$pgdata/postmaster.pid" \
|
||||
"$pgdata/postmaster.opts" \
|
||||
"$pgdata/recovery.signal" \
|
||||
"$pgdata/standby.signal" \
|
||||
"$pgdata/backup_label.old"
|
||||
rm -f "$pgdata/pg_wal/archive_status"/*.ready 2>/dev/null || :
|
||||
}
|
||||
|
||||
___normalize_backup_cleanup() {
|
||||
local tmpdir="$1"
|
||||
local pgdata="$2"
|
||||
if [ -n "$pgdata" ] && [ -d "$pgdata" ]; then
|
||||
pg_ctl -D "$pgdata" -m fast -w stop >/dev/null 2>&1 || :
|
||||
fi
|
||||
if [ -n "$tmpdir" ]; then
|
||||
rm -rf "$tmpdir"
|
||||
fi
|
||||
}
|
||||
|
||||
___normalize_backup_refresh_collation() {
|
||||
local sockdir="$1"
|
||||
local port="$2"
|
||||
local admin_role="$3"
|
||||
local database_name="$4"
|
||||
|
||||
psql -h "$sockdir" -p "$port" -U "$admin_role" -d postgres \
|
||||
-v ON_ERROR_STOP=1 -c "ALTER DATABASE \"$database_name\" REFRESH COLLATION VERSION;" \
|
||||
>/dev/null 2>&1 || :
|
||||
}
|
||||
|
||||
subcommand_normalize_backup() {
|
||||
change_namespace 'db normalize-backup'
|
||||
|
||||
NORMALIZE_INPUT_PATH=""
|
||||
NORMALIZE_OUTPUT_PATH="$DEFAULT_NORMALIZED_BACKUP_PATH"
|
||||
NORMALIZE_ROLE="$(id -un)"
|
||||
NORMALIZE_DATABASE="${PG_DATABASE:-testdb}"
|
||||
NORMALIZE_ADMIN_ROLE="postgres"
|
||||
NORMALIZE_PORT="${PG_PORT:-5432}"
|
||||
NORMALIZE_PRELOAD_LIBRARIES="${PG_SHARED_PRELOAD_LIBRARIES:-pg_cron}"
|
||||
NORMALIZE_DATABASE_EXPLICIT=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
help_normalize_backup
|
||||
exit 0
|
||||
;;
|
||||
-o|--output)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --output requires a path"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_OUTPUT_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--role)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --role requires a name"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_ROLE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--database)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --database requires a name"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_DATABASE="$2"
|
||||
NORMALIZE_DATABASE_EXPLICIT=1
|
||||
shift 2
|
||||
;;
|
||||
--admin-role)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --admin-role requires a name"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_ADMIN_ROLE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--*|-*)
|
||||
log error "normalize-backup argument $1 does not exist"
|
||||
exit 9
|
||||
;;
|
||||
*)
|
||||
if [ -n "$NORMALIZE_INPUT_PATH" ]; then
|
||||
log error "normalize-backup: unexpected argument $1"
|
||||
exit 9
|
||||
fi
|
||||
NORMALIZE_INPUT_PATH="$1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$NORMALIZE_INPUT_PATH" ]; then
|
||||
NORMALIZE_INPUT_PATH="$DEFAULT_BACKUP_PATH"
|
||||
fi
|
||||
if [ -z "$NORMALIZE_ROLE" ] || [ -z "$NORMALIZE_DATABASE" ] || [ -z "$NORMALIZE_ADMIN_ROLE" ]; then
|
||||
log error "normalize-backup: role, database, and admin role must be non-empty"
|
||||
exit 3
|
||||
fi
|
||||
if [ ! -d "$NORMALIZE_INPUT_PATH" ]; then
|
||||
log error "normalize-backup: input backup directory not found: $NORMALIZE_INPUT_PATH"
|
||||
exit 3
|
||||
fi
|
||||
if [ ! -f "$NORMALIZE_INPUT_PATH/base.tar.gz" ]; then
|
||||
log error "normalize-backup: missing $NORMALIZE_INPUT_PATH/base.tar.gz"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
NORMALIZE_INPUT_ABS=$(realpath "$NORMALIZE_INPUT_PATH")
|
||||
NORMALIZE_OUTPUT_ABS=$(realpath -m "$NORMALIZE_OUTPUT_PATH")
|
||||
|
||||
if [ "$NORMALIZE_INPUT_ABS" = "$NORMALIZE_OUTPUT_ABS" ]; then
|
||||
log error "normalize-backup: input and output paths must differ"
|
||||
exit 1
|
||||
fi
|
||||
case "$NORMALIZE_OUTPUT_ABS/" in
|
||||
"$NORMALIZE_INPUT_ABS"/*)
|
||||
log error "normalize-backup: output path must not be inside input backup"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
case "$NORMALIZE_INPUT_ABS/" in
|
||||
"$NORMALIZE_OUTPUT_ABS"/*)
|
||||
log error "normalize-backup: input backup must not be inside output path"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
mkdir -p "$(dirname "$NORMALIZE_OUTPUT_ABS")"
|
||||
|
||||
NORMALIZE_TMPDIR=$(mktemp -d)
|
||||
NORMALIZE_PGDATA="$NORMALIZE_TMPDIR/data"
|
||||
NORMALIZE_SOCKDIR="$NORMALIZE_TMPDIR/sock"
|
||||
NORMALIZE_LOG="$NORMALIZE_TMPDIR/postgres.log"
|
||||
trap '___normalize_backup_cleanup "$NORMALIZE_TMPDIR" "$NORMALIZE_PGDATA"' EXIT INT HUP
|
||||
|
||||
mkdir -m 700 "$NORMALIZE_PGDATA"
|
||||
mkdir -p "$NORMALIZE_SOCKDIR" "$NORMALIZE_PGDATA/pg_wal"
|
||||
|
||||
log notice "extracting backup into temporary PGDATA"
|
||||
tar -xzf "$NORMALIZE_INPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA"
|
||||
if [ -f "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" ]; then
|
||||
tar -xzf "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" -C "$NORMALIZE_PGDATA/pg_wal"
|
||||
fi
|
||||
|
||||
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
|
||||
|
||||
if [ ! -f "$NORMALIZE_PGDATA/postgresql.conf" ]; then
|
||||
cat > "$NORMALIZE_PGDATA/postgresql.conf" <<EOF
|
||||
listen_addresses = ''
|
||||
port = $NORMALIZE_PORT
|
||||
unix_socket_directories = '$NORMALIZE_SOCKDIR'
|
||||
logging_collector = off
|
||||
shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
|
||||
cron.database_name = '$NORMALIZE_DATABASE'
|
||||
cron.host = '$NORMALIZE_SOCKDIR'
|
||||
EOF
|
||||
fi
|
||||
cat > "$NORMALIZE_PGDATA/pg_hba.conf" <<EOF
|
||||
local all all trust
|
||||
EOF
|
||||
: > "$NORMALIZE_PGDATA/pg_ident.conf"
|
||||
cat > "$NORMALIZE_PGDATA/postgresql.auto.conf" <<EOF
|
||||
# Local-dev normalized backup overrides.
|
||||
listen_addresses = ''
|
||||
port = '$NORMALIZE_PORT'
|
||||
unix_socket_directories = '$NORMALIZE_SOCKDIR'
|
||||
logging_collector = 'off'
|
||||
hba_file = '$NORMALIZE_PGDATA/pg_hba.conf'
|
||||
ident_file = '$NORMALIZE_PGDATA/pg_ident.conf'
|
||||
shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
|
||||
cron.database_name = '$NORMALIZE_DATABASE'
|
||||
cron.host = '$NORMALIZE_SOCKDIR'
|
||||
EOF
|
||||
|
||||
log notice "starting temporary local cluster"
|
||||
with_closed_fds pg_ctl -D "$NORMALIZE_PGDATA" -o "-F" -w start > "$NORMALIZE_LOG" 2>&1 || {
|
||||
log error "normalize-backup: failed to start temporary cluster; see $NORMALIZE_LOG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" postgres
|
||||
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template1
|
||||
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template0
|
||||
|
||||
if [ "$NORMALIZE_DATABASE_EXPLICIT" -eq 0 ]; then
|
||||
detected_database=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||
-tAc "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate AND datname <> 'postgres' ORDER BY oid LIMIT 1" 2>/dev/null | sed '/^$/d' | head -n 1)
|
||||
if [ -n "$detected_database" ]; then
|
||||
NORMALIZE_DATABASE="$detected_database"
|
||||
fi
|
||||
fi
|
||||
|
||||
NORMALIZE_ROLE_SQL=$(___sql_literal "$NORMALIZE_ROLE")
|
||||
psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||
-v ON_ERROR_STOP=1 -c "DO \$\$ DECLARE v_role text := $NORMALIZE_ROLE_SQL; BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = v_role) THEN EXECUTE format('CREATE ROLE %I LOGIN SUPERUSER', v_role); ELSE EXECUTE format('ALTER ROLE %I LOGIN SUPERUSER', v_role); END IF; END \$\$;"
|
||||
|
||||
NORMALIZE_DATABASE_SQL=$(___sql_literal "$NORMALIZE_DATABASE")
|
||||
NORMALIZE_DATABASE_EXISTS=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||
-tAc "SELECT 1 FROM pg_database WHERE datname = $NORMALIZE_DATABASE_SQL" 2>/dev/null || true)
|
||||
if [ "$NORMALIZE_DATABASE_EXISTS" != "1" ]; then
|
||||
createdb -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" \
|
||||
-O "$NORMALIZE_ROLE" "$NORMALIZE_DATABASE"
|
||||
fi
|
||||
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" "$NORMALIZE_DATABASE"
|
||||
|
||||
log notice "stopping temporary local cluster"
|
||||
pg_ctl -D "$NORMALIZE_PGDATA" -m fast -w stop >> "$NORMALIZE_LOG" 2>&1
|
||||
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
|
||||
|
||||
log notice "writing normalized backup to $WHITE$NORMALIZE_OUTPUT_ABS$NC"
|
||||
rm -rf "$NORMALIZE_OUTPUT_ABS"
|
||||
mkdir -p "$NORMALIZE_OUTPUT_ABS"
|
||||
tar -czf "$NORMALIZE_OUTPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA" .
|
||||
printf '%s\n' "$NORMALIZE_DATABASE" > "$NORMALIZE_OUTPUT_ABS/database_name"
|
||||
|
||||
___normalize_backup_cleanup "$NORMALIZE_TMPDIR" ""
|
||||
trap - EXIT INT HUP
|
||||
restore_namespace
|
||||
}
|
||||
|
||||
subcommand_log() {
|
||||
change_namespace 'db log'
|
||||
: "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}"
|
||||
@@ -1175,6 +1456,7 @@ ___diff_dump_schema() {
|
||||
local port="$2"
|
||||
local output_file="$3"
|
||||
local tables="${4:-}"
|
||||
local database_name="${5:-${PG_DATABASE:-testdb}}"
|
||||
|
||||
log info "dumping schema to $WHITE$output_file$NC"
|
||||
|
||||
@@ -1189,17 +1471,17 @@ ___diff_dump_schema() {
|
||||
IFS="$old_IFS"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
pg_dump -h "$socket_dir" -p "$port" testdb \
|
||||
pg_dump -h "$socket_dir" -p "$port" "$database_name" \
|
||||
--schema-only --no-owner --no-privileges \
|
||||
$table_args > "$output_file" 2>/dev/null
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
pg_dump -h "$socket_dir" -p "$port" testdb \
|
||||
pg_dump -h "$socket_dir" -p "$port" "$database_name" \
|
||||
--data-only --no-owner --no-privileges \
|
||||
$table_args >> "$output_file" 2>/dev/null
|
||||
else
|
||||
# Schema only
|
||||
pg_dump -h "$socket_dir" -p "$port" testdb \
|
||||
pg_dump -h "$socket_dir" -p "$port" "$database_name" \
|
||||
--schema-only --no-owner --no-privileges \
|
||||
> "$output_file" 2>/dev/null
|
||||
fi
|
||||
@@ -1208,7 +1490,8 @@ ___diff_dump_schema() {
|
||||
___diff_immutable_tables() {
|
||||
local socket_dir="$1"
|
||||
local port="$2"
|
||||
psql -h "$socket_dir" -p "$port" -d testdb -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
|
||||
local database_name="${3:-${PG_DATABASE:-testdb}}"
|
||||
psql -h "$socket_dir" -p "$port" -d "$database_name" -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
|
||||
SELECT n.nspname || '.' || c.relname
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
@@ -1226,8 +1509,9 @@ ___diff_immutable_data() {
|
||||
local sock2="$3"
|
||||
local port2="$4"
|
||||
local out_file="$5"
|
||||
local database_name="${6:-${PG_DATABASE:-testdb}}"
|
||||
|
||||
if ! psql -h "$sock1" -p "$port1" -d testdb -tAc \
|
||||
if ! psql -h "$sock1" -p "$port1" -d "$database_name" -tAc \
|
||||
"SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='hectic' AND c.relname='immutable';" \
|
||||
>/dev/null 2>&1
|
||||
then
|
||||
@@ -1256,10 +1540,10 @@ ___diff_immutable_data() {
|
||||
log info " $tbl"
|
||||
: > "$data1"
|
||||
: > "$data2"
|
||||
pg_dump -h "$sock1" -p "$port1" testdb \
|
||||
pg_dump -h "$sock1" -p "$port1" "$database_name" \
|
||||
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
|
||||
> "$data1" 2>/dev/null || :
|
||||
pg_dump -h "$sock2" -p "$port2" testdb \
|
||||
pg_dump -h "$sock2" -p "$port2" "$database_name" \
|
||||
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
|
||||
> "$data2" 2>/dev/null || :
|
||||
{
|
||||
@@ -1405,6 +1689,8 @@ subcommand_diff() {
|
||||
DIFF_TABLES="" # TODO: add cron table
|
||||
DIFF_NO_CRON=0 # TODO: useless option
|
||||
DIFF_BACKUP_PATH=""
|
||||
DIFF_IGNORE_MIGRATION_FAIL=0
|
||||
DIFF_DATABASE="${PG_DATABASE:-testdb}"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
@@ -1420,6 +1706,10 @@ subcommand_diff() {
|
||||
DIFF_NO_CRON=1
|
||||
shift
|
||||
;;
|
||||
--ignore-migration-fail)
|
||||
DIFF_IGNORE_MIGRATION_FAIL=1
|
||||
shift
|
||||
;;
|
||||
-m|--mock)
|
||||
HYDRATE_USE_MOCK=1
|
||||
shift
|
||||
@@ -1433,9 +1723,8 @@ subcommand_diff() {
|
||||
exit 9
|
||||
;;
|
||||
*)
|
||||
# NOTE: yes, RESTORE, not DIFF prefix
|
||||
if [ -z "$RESTORE_BACKUP_PATH" ]; then
|
||||
RESTORE_BACKUP_PATH="$1"
|
||||
if [ -z "$DIFF_BACKUP_PATH" ]; then
|
||||
DIFF_BACKUP_PATH="$1"
|
||||
shift
|
||||
else
|
||||
log error "diff: unexpected argument $1"
|
||||
@@ -1445,14 +1734,21 @@ subcommand_diff() {
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$DIFF_BACKUP_PATH" ]; then
|
||||
DIFF_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
|
||||
fi
|
||||
if [ -f "$DIFF_BACKUP_PATH/database_name" ]; then
|
||||
DIFF_DATABASE=$(tr -d '\n' < "$DIFF_BACKUP_PATH/database_name")
|
||||
fi
|
||||
|
||||
DIFF_TMPDIR="${LOCAL_DIR}/focus/database-diff-operation"
|
||||
DIFF_PGDATA1="$DIFF_TMPDIR/pgdata1"
|
||||
DIFF_PGDATA2="$DIFF_TMPDIR/pgdata2"
|
||||
# TODO: suka, logi drugie
|
||||
DIFF_PGLOGFILE1="$DIFF_PGDATA1/logfile"
|
||||
DIFF_PGLOGFILE2="$DIFF_PGDATA2/logfile"
|
||||
DIFF_PGURL1="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA1/sock"
|
||||
DIFF_PGURL2="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA2/sock"
|
||||
DIFF_PGURL1="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA1/sock"
|
||||
DIFF_PGURL2="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA2/sock"
|
||||
|
||||
if [ "${DIFF_LOG+x}" ]; then
|
||||
log_pager -O "$DIFF_PGLOGFILE1" "$DIFF_PGLOGFILE2"
|
||||
@@ -1474,21 +1770,26 @@ subcommand_diff() {
|
||||
|
||||
log notice "provisioning ${WHITE}DB1$NC (backup + migrations)"
|
||||
|
||||
PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore
|
||||
PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore "$DIFF_BACKUP_PATH"
|
||||
|
||||
log info "applying migrations to ${WHITE}DB1$NC"
|
||||
subcommand_migrator migrate up all \
|
||||
--db-url \
|
||||
"$DIFF_PGURL1" \
|
||||
|| {
|
||||
log warn "migrations failed or none to apply"
|
||||
if [ "$DIFF_IGNORE_MIGRATION_FAIL" = "1" ]; then
|
||||
log warn "migrations failed; continuing because --ignore-migration-fail is set"
|
||||
else
|
||||
log error "migrations failed"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
log notice "provisioning ${WHITE}DB2$NC (current sources)"
|
||||
|
||||
log info "initializing ${WHITE}DB2$NC with postgres-init"
|
||||
PG_WORKING_DIR="$DIFF_PGDATA2" \
|
||||
PG_DATABASE="testdb" \
|
||||
PG_DATABASE="$DIFF_DATABASE" \
|
||||
PG_DISABLE_LOGGING=1 \
|
||||
postgres-init || {
|
||||
log error "failed to initialize ${WHITE}DB2$NC"
|
||||
@@ -1514,8 +1815,11 @@ subcommand_diff() {
|
||||
DIFF_DUMP1="$DIFF_TMPDIR/target.sql"
|
||||
DIFF_DUMP2="$DIFF_TMPDIR/source.sql"
|
||||
|
||||
___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES"
|
||||
___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES"
|
||||
___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES" "$DIFF_DATABASE"
|
||||
___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES" "$DIFF_DATABASE"
|
||||
|
||||
sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP1" || exit 1
|
||||
sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP2" || exit 1
|
||||
|
||||
# Optional: filter out cron tables
|
||||
if [ "$DIFF_NO_CRON" = "1" ]; then
|
||||
@@ -1541,7 +1845,8 @@ subcommand_diff() {
|
||||
___diff_immutable_data \
|
||||
"$DIFF_PGDATA1/sock" "5432" \
|
||||
"$DIFF_PGDATA2/sock" "5432" \
|
||||
"$DIFF_TMPDIR/diff"
|
||||
"$DIFF_TMPDIR/diff" \
|
||||
"$DIFF_DATABASE"
|
||||
data_status=$?
|
||||
|
||||
if [ "$schema_differs" = 0 ] && [ "$data_status" = 0 ]; then
|
||||
@@ -1573,6 +1878,14 @@ if ! [ "${AS_LIBRARY+x}" ]; then
|
||||
DB_URL=$2
|
||||
shift 2
|
||||
;;
|
||||
normalize-backup)
|
||||
if [ "${SUBCOMMAND+x}" ]; then
|
||||
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
||||
else
|
||||
SUBCOMMAND="normalize_backup"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init)
|
||||
if [ "${SUBCOMMAND+x}" ]; then
|
||||
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
||||
@@ -0,0 +1,222 @@
|
||||
# shellcheck shell=dash
|
||||
|
||||
: "${SCRIPT_NAME:=$(basename "$0")}"
|
||||
|
||||
help() {
|
||||
# shellcheck disable=SC2059
|
||||
printf "$(cat <<EOF
|
||||
${BGREEN}Usage:${NC} $SCRIPT_NAME [OPTIONS] <SUBCOMMAND> [OPTIONS]
|
||||
|
||||
PostgreSQL operations utility.
|
||||
|
||||
${BGREEN}Global Options:${NC}
|
||||
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
|
||||
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
|
||||
|
||||
${BGREEN}Subcommands:${NC}
|
||||
${BCYAN}secrets load${NC} [OPTIONS] Apply hectic bundle and load secrets
|
||||
|
||||
${BGREEN}Environment:${NC}
|
||||
${BBLACK}PGURL${NC} PostgreSQL connection string fallback
|
||||
${BBLACK}DB_URL${NC} PostgreSQL connection string fallback
|
||||
${BBLACK}HECTIC_DOTENV_CONTENT${NC} Raw dotenv content to load
|
||||
${BBLACK}HECTIC_DOTENV_FILE${NC} Dotenv file to read when readable
|
||||
${BBLACK}LOCAL_DIR${NC} Used with ENVIRONMENT for .env fallback
|
||||
${BBLACK}ENVIRONMENT${NC} Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
|
||||
|
||||
EOF
|
||||
)"
|
||||
}
|
||||
|
||||
help_secrets_load() {
|
||||
# shellcheck disable=SC2059
|
||||
printf "$(cat <<EOF
|
||||
${BGREEN}Usage:${NC} $SCRIPT_NAME ${BCYAN}secrets load${NC} [OPTIONS]
|
||||
|
||||
Apply the hectic SQL bundle and load dotenv-backed secrets into
|
||||
${BBLACK}hectic.secret${NC}.
|
||||
|
||||
${BGREEN}Options:${NC}
|
||||
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
|
||||
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
|
||||
${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} <path> Dotenv file path
|
||||
|
||||
${BGREEN}Resolution order:${NC}
|
||||
1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC}
|
||||
2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC}
|
||||
3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
|
||||
|
||||
Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv
|
||||
source can be resolved.
|
||||
|
||||
EOF
|
||||
)"
|
||||
}
|
||||
|
||||
resolve_pgurl() {
|
||||
if [ -n "${PGURL:-}" ]; then
|
||||
printf '%s' "$PGURL"
|
||||
elif [ -n "${DB_URL:-}" ]; then
|
||||
printf '%s' "$DB_URL"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_dotenv_content() {
|
||||
dotenv_file="${1:-}"
|
||||
|
||||
if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then
|
||||
printf '%s' "$HECTIC_DOTENV_CONTENT"
|
||||
elif [ -n "$dotenv_file" ]; then
|
||||
if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then
|
||||
return 2
|
||||
fi
|
||||
cat "$dotenv_file"
|
||||
elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
|
||||
cat "${LOCAL_DIR}/.env.${ENVIRONMENT}"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
subcommand_secrets_load() {
|
||||
change_namespace 'db ops secrets load'
|
||||
|
||||
dotenv_file="${HECTIC_DOTENV_FILE:-}"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
help_secrets_load
|
||||
restore_namespace
|
||||
exit 0
|
||||
;;
|
||||
-u|--url)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "missing value for $1"
|
||||
restore_namespace
|
||||
exit 3
|
||||
fi
|
||||
PGURL=$2
|
||||
DB_URL=$2
|
||||
shift 2
|
||||
;;
|
||||
-f|--dotenv-file)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "missing value for $1"
|
||||
restore_namespace
|
||||
exit 3
|
||||
fi
|
||||
dotenv_file=$2
|
||||
shift 2
|
||||
;;
|
||||
--*|-*)
|
||||
log error "secrets load argument $1 does not exist"
|
||||
restore_namespace
|
||||
exit 9
|
||||
;;
|
||||
*)
|
||||
log error "secrets load subcommand $1 does not exist"
|
||||
restore_namespace
|
||||
exit 9
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! pgurl="$(resolve_pgurl)"; then
|
||||
log error "PGURL or DB_URL is required"
|
||||
restore_namespace
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then
|
||||
:
|
||||
else
|
||||
dotenv_content_exit_code=$?
|
||||
if [ "$dotenv_content_exit_code" -eq 2 ]; then
|
||||
log error "dotenv file is not readable: $dotenv_file"
|
||||
else
|
||||
log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)"
|
||||
fi
|
||||
restore_namespace
|
||||
exit 3
|
||||
fi
|
||||
|
||||
log notice "apply hectic bundle and load secrets"
|
||||
apply_hectic_bundle "$pgurl" "$dotenv_content"
|
||||
restore_namespace
|
||||
}
|
||||
|
||||
subcommand_secrets() {
|
||||
change_namespace 'db ops secrets'
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
help_secrets_load
|
||||
restore_namespace
|
||||
exit 0
|
||||
fi
|
||||
|
||||
subcommand=$1
|
||||
shift
|
||||
|
||||
case $subcommand in
|
||||
load)
|
||||
subcommand_secrets_load "$@"
|
||||
;;
|
||||
-h|--help)
|
||||
help_secrets_load
|
||||
restore_namespace
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log error "secrets subcommand $subcommand does not exist"
|
||||
restore_namespace
|
||||
exit 9
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
help
|
||||
exit 0
|
||||
;;
|
||||
-u|--url)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "missing value for $1"
|
||||
exit 3
|
||||
fi
|
||||
PGURL=$2
|
||||
DB_URL=$2
|
||||
shift 2
|
||||
;;
|
||||
--*|-*)
|
||||
log error "argument $1 does not exist"
|
||||
exit 9
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
subcommand="${1:-}"
|
||||
|
||||
if [ -z "$subcommand" ]; then
|
||||
help
|
||||
exit 0
|
||||
fi
|
||||
|
||||
shift
|
||||
|
||||
case $subcommand in
|
||||
secrets)
|
||||
subcommand_secrets "$@"
|
||||
;;
|
||||
*)
|
||||
log error "subcommand $subcommand does not exist"
|
||||
exit 9
|
||||
;;
|
||||
esac
|
||||
+31
-24
@@ -1,32 +1,14 @@
|
||||
{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }:
|
||||
let
|
||||
shell = "${dash}/bin/dash";
|
||||
|
||||
hecticInheritanceSqlPath = ../../lib/hook/sql/hectic-inheritance.sql;
|
||||
|
||||
hecticInheritance = runCommand "hectic-inheritance" { } ''
|
||||
mkdir -p "$out/share/hectic"
|
||||
cp ${hecticInheritanceSqlPath} "$out/share/hectic/hectic-inheritance.sql"
|
||||
cp ${self.lib.hectic.inheritance.path} "$out/share/hectic/hectic-inheritance.sql"
|
||||
'';
|
||||
|
||||
# Materialize the templated version SQL into the Nix store as a real file
|
||||
# so it can be passed by path to psql -f (alongside the static siblings).
|
||||
hecticVersionSqlFile = pkgs-writeText "hectic-version.sql" self.lib.hectic.version.sql;
|
||||
pkgs-writeText = name: text: runCommand name { inherit text; passAsFile = [ "text" ]; } ''
|
||||
cp "$textPath" "$out"
|
||||
'';
|
||||
applyBundle = self.lib.hectic.applyBundleScript;
|
||||
|
||||
hecticEnv = ''
|
||||
HECTIC_VERSION_SQL=${hecticVersionSqlFile}
|
||||
HECTIC_SECRET_SQL=${self.lib.hectic.secret.path}
|
||||
HECTIC_MIGRATION_SQL=${self.lib.hectic.migration.path}
|
||||
HECTIC_INHERITANCE_SQL=${self.lib.hectic.inheritance.path}
|
||||
export HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL
|
||||
'';
|
||||
|
||||
applyBundle = builtins.readFile self.lib.hectic.applyBundleScript;
|
||||
|
||||
mkDatabase =
|
||||
mkDbDev =
|
||||
{ postgresql ? postgresql_17 }:
|
||||
hectic.writeShellApplication {
|
||||
inherit shell;
|
||||
@@ -44,9 +26,8 @@ let
|
||||
${builtins.readFile hectic.helpers.posix-shell.quote}
|
||||
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
|
||||
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
|
||||
${hecticEnv}
|
||||
${applyBundle}
|
||||
${builtins.readFile ./database.sh}
|
||||
${builtins.readFile ./db-dev.sh}
|
||||
'';
|
||||
|
||||
meta = {
|
||||
@@ -55,6 +36,31 @@ let
|
||||
};
|
||||
};
|
||||
|
||||
mkDbOps =
|
||||
{ postgresql ? postgresql_17 }:
|
||||
hectic.writeShellApplication {
|
||||
inherit shell;
|
||||
bashOptions = [
|
||||
"errexit"
|
||||
"nounset"
|
||||
];
|
||||
excludeShellChecks = [ "SC2209" ];
|
||||
name = "db-ops";
|
||||
runtimeInputs = [ postgresql coreutils ];
|
||||
|
||||
text = ''
|
||||
${builtins.readFile hectic.helpers.posix-shell.log}
|
||||
${builtins.readFile hectic.helpers.posix-shell.change_namespace}
|
||||
${applyBundle}
|
||||
${builtins.readFile ./db-ops.sh}
|
||||
'';
|
||||
|
||||
meta = {
|
||||
description = "PostgreSQL operations utility";
|
||||
mainProgram = "db-ops";
|
||||
};
|
||||
};
|
||||
|
||||
mkPostgresInit =
|
||||
{ postgresql ? postgresql_17 }:
|
||||
hectic.writeShellApplication {
|
||||
@@ -92,7 +98,8 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
"db-tool" = lib.makeOverridable mkDatabase { };
|
||||
"db-dev" = lib.makeOverridable mkDbDev { };
|
||||
"db-ops" = lib.makeOverridable mkDbOps { };
|
||||
"postgres-init" = lib.makeOverridable mkPostgresInit { };
|
||||
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
|
||||
"hectic-inheritance" = hecticInheritance;
|
||||
|
||||
@@ -1,9 +1,46 @@
|
||||
#!/bin/dash
|
||||
|
||||
# Stop a local PostgreSQL cluster started by postgres-init.
|
||||
#
|
||||
# Public contract:
|
||||
# - Accepts PG_WORKING_DIR directly, or derives it from LOCAL_DIR.
|
||||
# - If no cluster root can be resolved, exits successfully without doing
|
||||
# anything. This keeps cleanup safe in traps and partial-init flows.
|
||||
# - If no postmaster.pid exists, treats the cluster as already stopped.
|
||||
# - NO_TTY=1 redirects pg_ctl output into a temp log file instead of writing to
|
||||
# the current terminal.
|
||||
|
||||
postgres_cleanup_help() {
|
||||
cat <<'EOF'
|
||||
Usage: postgres-cleanup
|
||||
|
||||
Stop a local PostgreSQL cluster if it is running.
|
||||
|
||||
Environment:
|
||||
PG_WORKING_DIR Cluster root directory
|
||||
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
|
||||
NO_TTY Redirect pg_ctl output to a temp file
|
||||
|
||||
Behavior:
|
||||
- Returns success if the cluster directory cannot be resolved.
|
||||
- Returns success if postmaster.pid is already absent.
|
||||
- Ignores pg_ctl stop errors so cleanup remains trap-safe.
|
||||
EOF
|
||||
}
|
||||
|
||||
postgres_cleanup_main() {
|
||||
case "${1:-}" in
|
||||
-h|--help)
|
||||
postgres_cleanup_help
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
|
||||
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
|
||||
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
|
||||
# Cleanup is intentionally forgiving so it can be used in traps after
|
||||
# partial startup failures without masking the real error.
|
||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
|
||||
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
|
||||
|
||||
@@ -1,6 +1,58 @@
|
||||
#!/bin/dash
|
||||
|
||||
# Initialize or reuse a local PostgreSQL cluster for development workflows.
|
||||
#
|
||||
# Public contract:
|
||||
# - Requires either PG_WORKING_DIR or LOCAL_DIR.
|
||||
# - Produces a ready-to-use database and exports:
|
||||
# POSTGRESQL_HOST, POSTGRESQL_PORT, POSTGRESQL_USER, POSTGRESQL_DATABASE,
|
||||
# PGURL, and also the variable named by PG_URL_VAR.
|
||||
# - Reuses an existing cluster only when PG_REUSE is set and PG_VERSION exists.
|
||||
# - If a reused cluster cannot start, it is reinitialized automatically.
|
||||
#
|
||||
# Important knobs:
|
||||
# - PG_WORKING_DIR: cluster root (defaults to $LOCAL_DIR/focus/postgresql)
|
||||
# - PG_DATABASE: database to create/ensure (default: testdb)
|
||||
# - PG_PORT: socket port (default: 5432)
|
||||
# - PG_URL_VAR: alternate URL variable to export in addition to PGURL
|
||||
# - PG_CONF_FILE: base postgresql.conf to copy on fresh init
|
||||
# - PG_SHARED_PRELOAD_LIBRARIES: preload list; empty string disables pg_cron
|
||||
# - PG_DISABLE_LOGGING=1: disable logging_collector in generated config
|
||||
# - NO_TTY=1: redirect pg_ctl stop/start output into temp log files
|
||||
|
||||
postgres_init_help() {
|
||||
cat <<'EOF'
|
||||
Usage: postgres-init
|
||||
|
||||
Initialize or reuse a local PostgreSQL cluster.
|
||||
|
||||
Environment:
|
||||
PG_WORKING_DIR Cluster root directory
|
||||
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
|
||||
PG_DATABASE Database name to create/ensure (default: testdb)
|
||||
PG_PORT PostgreSQL port / unix socket port (default: 5432)
|
||||
PG_URL_VAR Extra URL variable to export alongside PGURL
|
||||
PG_CONF_FILE Base postgresql.conf copied on fresh init only
|
||||
PG_SHARED_PRELOAD_LIBRARIES Preload list; empty disables pg_cron preload
|
||||
PG_DISABLE_LOGGING Set to 1 to disable logging_collector
|
||||
PG_REUSE Reuse existing cluster if PG_VERSION exists
|
||||
NO_TTY Redirect pg_ctl output to temp files
|
||||
|
||||
Behavior:
|
||||
- Rewrites runtime socket/port/cron settings on every launch.
|
||||
- Creates the target database if missing.
|
||||
- If a reused cluster exists but cannot start, reinitializes it automatically.
|
||||
EOF
|
||||
}
|
||||
|
||||
postgres_init_main() {
|
||||
case "${1:-}" in
|
||||
-h|--help)
|
||||
postgres_init_help
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
|
||||
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
|
||||
return 1
|
||||
@@ -25,8 +77,87 @@ postgres_init_main() {
|
||||
fi
|
||||
mkdir -p "$sockdir" || return 1
|
||||
|
||||
# Reuse is optimistic: if a cluster exists on disk, try to start it first.
|
||||
# We only destroy state after a real startup failure proves the cluster is
|
||||
# broken, which keeps long-lived local DBs intact when possible.
|
||||
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
|
||||
|
||||
if [ "$PG_REUSE" -eq 0 ]; then
|
||||
rm -rf "$data" "$sockdir" || return 1
|
||||
mkdir -p "$sockdir" || return 1
|
||||
initdb -D "$data" --no-locale -E UTF8 || return 1
|
||||
if [ -n "${PG_CONF_FILE:-}" ]; then
|
||||
# PG_CONF_FILE replaces the base postgresql.conf on fresh init only. We
|
||||
# still append runtime values later so the helper can relocate sockets,
|
||||
# ports, and cron settings regardless of the custom file contents.
|
||||
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
||||
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
||||
else
|
||||
{
|
||||
printf '%s\n' "listen_addresses = ''"
|
||||
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||
printf '%s\n' 'logging_collector = on'
|
||||
printf '%s\n' "log_directory = 'log'"
|
||||
fi
|
||||
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||
printf '%s\n' "cron.database_name = '$db'"
|
||||
printf '%s\n' "cron.host = '$sockdir'"
|
||||
fi
|
||||
} >> "$data/postgresql.conf" || return 1
|
||||
fi
|
||||
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
||||
fi
|
||||
|
||||
# Rewrite the runtime knobs on every launch. Reused clusters may have been
|
||||
# started from another workspace/session, so we must override stale socket,
|
||||
# port, and pg_cron wiring before attempting startup.
|
||||
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
||||
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
||||
|
||||
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
|
||||
{
|
||||
printf '%s\n' "port = '$PG_PORT'"
|
||||
printf '%s\n' "unix_socket_directories = '$sockdir'"
|
||||
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
|
||||
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
|
||||
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||
printf '%s\n' "logging_collector = 'on'"
|
||||
else
|
||||
printf '%s\n' "logging_collector = 'off'"
|
||||
fi
|
||||
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||
printf '%s\n' "cron.database_name = '$db'"
|
||||
printf '%s\n' "cron.host = '$sockdir'"
|
||||
fi
|
||||
} >> "$data/postgresql.auto.conf" || return 1
|
||||
|
||||
_pg_start_exit=0
|
||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
|
||||
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
||||
else
|
||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
|
||||
fi
|
||||
|
||||
if [ "$_pg_start_exit" -ne 0 ]; then
|
||||
if [ "$PG_REUSE" -eq 1 ]; then
|
||||
# Self-heal path: the on-disk cluster exists but cannot complete startup
|
||||
# (for example bad WAL / invalid checkpoint). At this point preserving
|
||||
# the broken state is less useful than reinitializing automatically.
|
||||
printf '%s\n' "postgres-init: reused cluster failed to start; reinitializing $wd" >&2
|
||||
PG_REUSE=0
|
||||
rm -rf "$data" "$sockdir" || return 1
|
||||
mkdir -p "$sockdir" || return 1
|
||||
initdb -D "$data" --no-locale -E UTF8 || return 1
|
||||
@@ -34,20 +165,61 @@ postgres_init_main() {
|
||||
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
||||
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
||||
else
|
||||
{ printf '%s\n' "listen_addresses = ''"; [ "$PG_DISABLE_LOGGING" -eq 0 ] && { printf '%s\n' 'logging_collector = on'; printf '%s\n' "log_directory = 'log'"; }; [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ] && { printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"; printf '%s\n' "cron.database_name = '$db'"; printf '%s\n' "cron.host = '$sockdir'"; }; :; } >> "$data/postgresql.conf" || return 1
|
||||
{
|
||||
printf '%s\n' "listen_addresses = ''"
|
||||
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||
printf '%s\n' 'logging_collector = on'
|
||||
printf '%s\n' "log_directory = 'log'"
|
||||
fi
|
||||
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||
printf '%s\n' "cron.database_name = '$db'"
|
||||
printf '%s\n' "cron.host = '$sockdir'"
|
||||
fi
|
||||
} >> "$data/postgresql.conf" || return 1
|
||||
fi
|
||||
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
||||
fi
|
||||
|
||||
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
||||
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
||||
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
|
||||
{
|
||||
printf '%s\n' "port = '$PG_PORT'"
|
||||
printf '%s\n' "unix_socket_directories = '$sockdir'"
|
||||
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
|
||||
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
|
||||
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||
printf '%s\n' "logging_collector = 'on'"
|
||||
else
|
||||
printf '%s\n' "logging_collector = 'off'"
|
||||
fi
|
||||
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||
printf '%s\n' "cron.database_name = '$db'"
|
||||
printf '%s\n' "cron.host = '$sockdir'"
|
||||
fi
|
||||
} >> "$data/postgresql.auto.conf" || return 1
|
||||
_pg_start_exit=0
|
||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || return 2
|
||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
|
||||
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
||||
else
|
||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || return 2
|
||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
|
||||
fi
|
||||
[ "$_pg_start_exit" -eq 0 ] || return 2
|
||||
printf '%s\n' "postgres-init: reinitialized broken cluster at $wd" >&2
|
||||
else
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
|
||||
user="$(id -un)" || return 1
|
||||
@@ -61,9 +233,13 @@ postgres_init_main() {
|
||||
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
|
||||
|
||||
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
|
||||
_pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
|
||||
# Export both names for compatibility: some callers consume plain PGURL,
|
||||
# while multi-cluster shells also need a project-specific variable like
|
||||
# WIPE_PGURL or BMSEARCH_PGURL in the same environment.
|
||||
PGURL="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
|
||||
export PGURL
|
||||
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
|
||||
export "${PG_URL_VAR}=${_pg_url}" || return 1
|
||||
export "${PG_URL_VAR}=${PGURL}" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
+8
-1
@@ -123,6 +123,7 @@ in {
|
||||
py3-swifter = pkgs.callPackage ./py3-swifter.nix {};
|
||||
py3-aiogram-newsletter = pkgs.callPackage ./py3-aiogram-newsletter.nix {};
|
||||
py3-openai-shap-e = pkgs.callPackage ./py3-openai-shap-e.nix {};
|
||||
hiddify-core = pkgs.callPackage ./hiddify-core {};
|
||||
nvim-alias = pkgs.callPackage ./nvim-alias.nix {};
|
||||
bolt-unpack = pkgs.callPackage ./bolt-unpack.nix {};
|
||||
merge-archive = pkgs.callPackage ./merge-archive {};
|
||||
@@ -131,6 +132,7 @@ in {
|
||||
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
|
||||
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
|
||||
migration-name = pkgs.callPackage ./migration-name.nix {};
|
||||
plgo = pkgs.callPackage ./plgo {};
|
||||
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
|
||||
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
|
||||
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
|
||||
@@ -141,14 +143,18 @@ in {
|
||||
watch = pkgs.callPackage ./c/watch/default.nix {};
|
||||
support-bot = pkgs.callPackage ./support-bot {};
|
||||
gitea-heatmap = pkgs.callPackage ./gitea {};
|
||||
gitea-runner-nix-image = pkgs.callPackage ./gitea-runner-nix-image {};
|
||||
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
|
||||
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
|
||||
deploy = pkgs.callPackage ./deploy { inherit inputs; };
|
||||
element-web = pkgs.callPackage ./element-web {};
|
||||
shellplot = pkgs.callPackage ./shellplot {};
|
||||
which-country-rs = pkgs.callPackage ./which-country-rs {};
|
||||
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
|
||||
migrator = pkgs.callPackage ./migrator { inherit self; };
|
||||
"parse-uri" = pkgs.callPackage ./parse-uri {};
|
||||
"db-tool" = dbToolPkgs."db-tool";
|
||||
"db-dev" = dbToolPkgs."db-dev";
|
||||
"db-ops" = dbToolPkgs."db-ops";
|
||||
"postgres-init" = dbToolPkgs."postgres-init";
|
||||
"postgres-cleanup" = dbToolPkgs."postgres-cleanup";
|
||||
"hectic-inheritance" = dbToolPkgs."hectic-inheritance";
|
||||
@@ -170,5 +176,6 @@ in {
|
||||
pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
|
||||
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
|
||||
pg-15-ext-plsh = buildPlShExt pkgs "15";
|
||||
stable-video-diffusion-api = pkgs.callPackage ./stable-video-diffusion-api {};
|
||||
media-browser = pkgs.callPackage ./media-browser {};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
fetchFromGitHub,
|
||||
jq,
|
||||
nodejs,
|
||||
jitsi-meet,
|
||||
fetchPnpmDeps,
|
||||
pnpm_10,
|
||||
pnpmConfigHook,
|
||||
faketty,
|
||||
config,
|
||||
conf ? config.element-web.conf or { },
|
||||
}:
|
||||
|
||||
let
|
||||
pnpm = pnpm_10;
|
||||
patchDir = ./patches;
|
||||
patches = if builtins.pathExists patchDir then map (name: patchDir + "/${name}") (
|
||||
builtins.filter (name: lib.hasSuffix ".patch" name) (builtins.attrNames (builtins.readDir patchDir))
|
||||
) else [ ];
|
||||
noPhoningHome = {
|
||||
disable_guests = true;
|
||||
};
|
||||
jitsi-meet-override = jitsi-meet.overrideAttrs (previousAttrs: {
|
||||
meta = removeAttrs previousAttrs.meta [ "knownVulnerabilities" ];
|
||||
});
|
||||
element-web-unwrapped = stdenv.mkDerivation (finalAttrs: {
|
||||
pname = "element-web";
|
||||
version = "1.12.20";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "element-hq";
|
||||
repo = "element-web";
|
||||
tag = "v${finalAttrs.version}";
|
||||
hash = "sha256-pbzuPgKJ0DmrDSTO7ZTDArX+Xr9k/ndAGZvQg2kMTMQ=";
|
||||
};
|
||||
|
||||
#inherit patches; #WIP
|
||||
|
||||
pnpmDeps = fetchPnpmDeps {
|
||||
pname = "element";
|
||||
inherit (finalAttrs) version src;
|
||||
inherit pnpm;
|
||||
fetcherVersion = 3;
|
||||
hash = "sha256-snm7vaHCVX6vYrkmsz5HlYMKx5Ks3K9jvUinkJ41CU0=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
jq
|
||||
nodejs
|
||||
pnpm
|
||||
pnpmConfigHook
|
||||
faketty
|
||||
];
|
||||
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
|
||||
cd apps/web
|
||||
|
||||
export VERSION=${finalAttrs.version}
|
||||
faketty pnpm run build
|
||||
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
cp -R webapp $out
|
||||
cp ${jitsi-meet-override}/libs/external_api.min.js $out/jitsi_external_api.min.js
|
||||
echo "${finalAttrs.version}" > "$out/version"
|
||||
jq -s '.[0] * $conf' "config.sample.json" --argjson "conf" '${builtins.toJSON noPhoningHome}' > "$out/config.json"
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
meta = {
|
||||
description = "Glossy Matrix collaboration client for the web";
|
||||
homepage = "https://element.io/";
|
||||
changelog = "https://github.com/element-hq/element-web/blob/v${finalAttrs.version}/CHANGELOG.md";
|
||||
teams = [ lib.teams.matrix ];
|
||||
license = lib.licenses.agpl3Plus;
|
||||
platforms = lib.platforms.all;
|
||||
};
|
||||
});
|
||||
in
|
||||
if conf == { } then
|
||||
element-web-unwrapped
|
||||
else
|
||||
stdenv.mkDerivation {
|
||||
pname = "${element-web-unwrapped.pname}-wrapped";
|
||||
inherit (element-web-unwrapped) version meta;
|
||||
|
||||
dontUnpack = true;
|
||||
|
||||
nativeBuildInputs = [ jq ];
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
mkdir -p $out
|
||||
ln -s ${element-web-unwrapped}/* $out
|
||||
rm $out/config.json
|
||||
jq -s '.[0] * $conf' "${element-web-unwrapped}/config.json" --argjson "conf" ${lib.escapeShellArg (builtins.toJSON conf)} > "$out/config.json"
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
inherit conf;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
diff --git a/apps/web/src/IConfigOptions.ts b/apps/web/src/IConfigOptions.ts
|
||||
index a3a6a32..c61c24f 100644
|
||||
--- a/apps/web/src/IConfigOptions.ts
|
||||
+++ b/apps/web/src/IConfigOptions.ts
|
||||
@@ -105,6 +105,15 @@ export interface IConfigOptions {
|
||||
show_labs_settings: boolean;
|
||||
features?: Record<string, boolean>; // <FeatureName, EnabledBool>
|
||||
|
||||
+ hectic?: {
|
||||
+ // This local package config intentionally uses the documented camelCase path
|
||||
+ // `hectic.videoMessages.enabled`.
|
||||
+ // eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
+ videoMessages?: {
|
||||
+ enabled?: boolean;
|
||||
+ };
|
||||
+ };
|
||||
+
|
||||
/**
|
||||
* Bug report endpoint URL. "local" means the logs should not be uploaded.
|
||||
*/
|
||||
diff --git a/apps/web/src/SdkConfig.ts b/apps/web/src/SdkConfig.ts
|
||||
index 4be6464..9f48743 100644
|
||||
--- a/apps/web/src/SdkConfig.ts
|
||||
+++ b/apps/web/src/SdkConfig.ts
|
||||
@@ -28,6 +28,12 @@ export const DEFAULTS: DeepReadonly<IConfigOptions> = {
|
||||
integrations_ui_url: "https://scalar.vector.im/",
|
||||
integrations_rest_url: "https://scalar.vector.im/api",
|
||||
show_labs_settings: false,
|
||||
+ hectic: {
|
||||
+ // eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
+ videoMessages: {
|
||||
+ enabled: false,
|
||||
+ },
|
||||
+ },
|
||||
force_verification: false,
|
||||
|
||||
jitsi: {
|
||||
diff --git a/apps/web/src/components/views/rooms/MessageComposer.tsx b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||
index 06c843f..eea76b7 100644
|
||||
--- a/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||
+++ b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||
@@ -33,6 +33,7 @@ import ReplyPreview from "./ReplyPreview";
|
||||
import { UserIdentityWarning } from "./UserIdentityWarning";
|
||||
import { UPDATE_EVENT } from "../../../stores/AsyncStore";
|
||||
import VoiceRecordComposerTile from "./VoiceRecordComposerTile";
|
||||
+import VideoRecordComposerTile from "./VideoRecordComposerTile";
|
||||
import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore";
|
||||
import { RecordingState } from "../../../audio/VoiceRecording";
|
||||
import type ResizeNotifier from "../../../utils/ResizeNotifier";
|
||||
@@ -92,6 +93,7 @@ interface IState {
|
||||
composerContent: string;
|
||||
isComposerEmpty: boolean;
|
||||
haveRecording: boolean;
|
||||
+ haveVideoRecording: boolean;
|
||||
recordingTimeLeftSeconds?: number;
|
||||
me?: RoomMember;
|
||||
isMenuOpen: boolean;
|
||||
@@ -113,6 +115,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
private dispatcherRef?: string;
|
||||
private messageComposerInput = createRef<SendMessageComposerClass>();
|
||||
private voiceRecordingButton = createRef<VoiceRecordComposerTile>();
|
||||
+ private videoRecordingButton = createRef<VideoRecordComposerTile>();
|
||||
private ref = createRef<HTMLDivElement>();
|
||||
private instanceId: number;
|
||||
|
||||
@@ -144,6 +147,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
isComposerEmpty: initialComposerContent?.length === 0,
|
||||
composerContent: initialComposerContent,
|
||||
haveRecording: false,
|
||||
+ haveVideoRecording: false,
|
||||
recordingTimeLeftSeconds: undefined, // when set to a number, shows a toast
|
||||
isMenuOpen: false,
|
||||
isStickerPickerOpen: false,
|
||||
@@ -526,6 +530,18 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
}
|
||||
};
|
||||
|
||||
+ private onVideoRecordStartClick = (): void => {
|
||||
+ this.videoRecordingButton.current?.onRecordStartEndClick();
|
||||
+
|
||||
+ if (this.context.narrow) {
|
||||
+ this.toggleButtonMenu();
|
||||
+ }
|
||||
+ };
|
||||
+
|
||||
+ private onVideoRecordingStateChange = (haveVideoRecording: boolean): void => {
|
||||
+ this.setState({ haveVideoRecording });
|
||||
+ };
|
||||
+
|
||||
public render(): React.ReactNode {
|
||||
let leftIcon: false | JSX.Element = false;
|
||||
if (!this.state.isWysiwygLabEnabled) {
|
||||
@@ -561,13 +577,14 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
const menuPosition = this.getMenuPosition();
|
||||
|
||||
const canSendMessages = this.context.canSendMessages && !this.context.tombstone;
|
||||
+ const hasActiveRecording = this.state.haveRecording || this.state.haveVideoRecording;
|
||||
let composer: ReactNode;
|
||||
if (canSendMessages) {
|
||||
if (this.state.isWysiwygLabEnabled && menuPosition) {
|
||||
composer = (
|
||||
<SendWysiwygComposer
|
||||
key="controls_input"
|
||||
- disabled={this.state.haveRecording}
|
||||
+ disabled={hasActiveRecording}
|
||||
onChange={this.onWysiwygChange}
|
||||
onSend={this.sendMessage}
|
||||
isRichTextEnabled={this.state.isRichTextEnabled}
|
||||
@@ -588,7 +605,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
relation={this.props.relation}
|
||||
replyToEvent={this.props.replyToEvent}
|
||||
onChange={this.onChange}
|
||||
- disabled={this.state.haveRecording}
|
||||
+ disabled={hasActiveRecording}
|
||||
toggleStickerPickerOpen={this.toggleStickerPickerOpen}
|
||||
/>
|
||||
);
|
||||
@@ -608,6 +625,11 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
replyToEvent={this.props.replyToEvent}
|
||||
/>
|
||||
</Tooltip>,
|
||||
+ <VideoRecordComposerTile
|
||||
+ key="controls_video_record"
|
||||
+ ref={this.videoRecordingButton}
|
||||
+ onRecordingStateChange={this.onVideoRecordingStateChange}
|
||||
+ />,
|
||||
);
|
||||
} else if (this.context.tombstone) {
|
||||
const replacementRoomId = this.context.tombstone.getContent()["replacement_room"];
|
||||
@@ -688,12 +710,13 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
{canSendMessages && (
|
||||
<MessageComposerButtons
|
||||
addEmoji={this.addEmoji}
|
||||
- haveRecording={this.state.haveRecording}
|
||||
+ haveRecording={hasActiveRecording}
|
||||
isMenuOpen={this.state.isMenuOpen}
|
||||
isStickerPickerOpen={this.state.isStickerPickerOpen}
|
||||
menuPosition={menuPosition}
|
||||
relation={this.props.relation}
|
||||
onRecordStartEndClick={this.onRecordStartEndClick}
|
||||
+ onVideoRecordStartClick={this.onVideoRecordStartClick}
|
||||
setStickerPickerOpen={this.setStickerPickerOpen}
|
||||
showLocationButton={
|
||||
!window.electron && SettingsStore.getValue(UIFeature.LocationSharing)
|
||||
diff --git a/apps/web/src/components/views/rooms/MessageComposerButtons.tsx b/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
|
||||
index 6f4f5d1..0c1b7ff 100644
|
||||
--- a/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
|
||||
+++ b/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
|
||||
@@ -44,6 +44,8 @@ import { useSettingValue } from "../../../hooks/useSettings";
|
||||
import AccessibleButton, { type ButtonEvent } from "../elements/AccessibleButton";
|
||||
import { useScopedRoomContext } from "../../../contexts/ScopedRoomContext.tsx";
|
||||
import { useRoomUploadViewModel } from "../../../viewmodels/room/RoomUploadViewModel.tsx";
|
||||
+import SdkConfig from "../../../SdkConfig";
|
||||
+import { isVideoMessageRecorderSupported } from "./VideoRecordComposerTile";
|
||||
|
||||
interface IProps {
|
||||
addEmoji: (emoji: string) => boolean;
|
||||
@@ -52,6 +54,7 @@ interface IProps {
|
||||
isStickerPickerOpen: boolean;
|
||||
menuPosition?: MenuProps;
|
||||
onRecordStartEndClick: () => void;
|
||||
+ onVideoRecordStartClick: () => void;
|
||||
relation?: IEventRelation;
|
||||
setStickerPickerOpen: (isStickerPickerOpen: boolean) => void;
|
||||
showLocationButton: boolean;
|
||||
@@ -103,6 +106,7 @@ const MessageComposerButtons: React.FC<IProps> = (props: IProps) => {
|
||||
)),
|
||||
showStickersButton(props),
|
||||
voiceRecordingButton(props, narrow),
|
||||
+ videoRecordingButton(props, narrow),
|
||||
props.showPollsButton ? pollButton(room, props.relation) : null,
|
||||
showLocationButton(props, room, matrixClient),
|
||||
];
|
||||
@@ -122,6 +126,7 @@ const MessageComposerButtons: React.FC<IProps> = (props: IProps) => {
|
||||
moreButtons = [
|
||||
showStickersButton(props),
|
||||
voiceRecordingButton(props, narrow),
|
||||
+ videoRecordingButton(props, narrow),
|
||||
props.showPollsButton ? pollButton(room, props.relation) : null,
|
||||
showLocationButton(props, room, matrixClient),
|
||||
];
|
||||
@@ -203,6 +208,25 @@ function voiceRecordingButton(props: IProps, narrow: boolean): ReactElement | nu
|
||||
);
|
||||
}
|
||||
|
||||
+function videoRecordingButton(props: IProps, narrow: boolean): ReactElement | null {
|
||||
+ // The current recorder skeleton follows the voice recorder and stays out of narrow mode.
|
||||
+ if (narrow || SdkConfig.get("hectic")?.videoMessages?.enabled !== true || !isVideoMessageRecorderSupported()) {
|
||||
+ return null;
|
||||
+ }
|
||||
+
|
||||
+ return (
|
||||
+ <CollapsibleButton
|
||||
+ key="video_message_send"
|
||||
+ className="mx_MessageComposer_button"
|
||||
+ data-testid="video-message-button"
|
||||
+ onClick={props.onVideoRecordStartClick}
|
||||
+ title={_t("composer|video_message_button")}
|
||||
+ >
|
||||
+ <span aria-hidden="true">●</span>
|
||||
+ </CollapsibleButton>
|
||||
+ );
|
||||
+}
|
||||
+
|
||||
function pollButton(room: Room, relation?: IEventRelation): ReactElement {
|
||||
return <PollButton key="polls" room={room} relation={relation} />;
|
||||
}
|
||||
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
new file mode 100644
|
||||
index 0000000..c5f0adc
|
||||
--- /dev/null
|
||||
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
@@ -0,0 +1,249 @@
|
||||
+/*
|
||||
+Copyright 2026 Element Creations Ltd.
|
||||
+
|
||||
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
+Please see LICENSE files in the repository root for full details.
|
||||
+*/
|
||||
+
|
||||
+import React, { type ReactNode } from "react";
|
||||
+import { logger } from "matrix-js-sdk/src/logger";
|
||||
+import { DeleteIcon, SendSolidIcon, StopSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
+
|
||||
+import { _t } from "../../../languageHandler";
|
||||
+import AccessibleButton from "../elements/AccessibleButton";
|
||||
+
|
||||
+interface IProps {
|
||||
+ onRecordingStateChange: (haveVideoRecording: boolean) => void;
|
||||
+}
|
||||
+
|
||||
+interface IState {
|
||||
+ error?: string;
|
||||
+ isRecording: boolean;
|
||||
+ previewUrl?: string;
|
||||
+}
|
||||
+
|
||||
+export function isVideoMessageRecorderSupported(): boolean {
|
||||
+ return (
|
||||
+ typeof window !== "undefined" &&
|
||||
+ typeof MediaRecorder !== "undefined" &&
|
||||
+ typeof navigator !== "undefined" &&
|
||||
+ !!navigator.mediaDevices?.getUserMedia
|
||||
+ );
|
||||
+}
|
||||
+
|
||||
+/**
|
||||
+ * Container tile for rendering the config-gated video message recorder skeleton in the composer.
|
||||
+ */
|
||||
+export default class VideoRecordComposerTile extends React.PureComponent<IProps, IState> {
|
||||
+ private chunks: Blob[] = [];
|
||||
+ private mediaRecorder?: MediaRecorder;
|
||||
+ private stream?: MediaStream;
|
||||
+ private isRequestingMedia = false;
|
||||
+ private isUnmounted = false;
|
||||
+ private shouldDiscardRecording = false;
|
||||
+
|
||||
+ public constructor(props: IProps) {
|
||||
+ super(props);
|
||||
+
|
||||
+ this.state = {
|
||||
+ isRecording: false,
|
||||
+ };
|
||||
+ }
|
||||
+
|
||||
+ public componentWillUnmount(): void {
|
||||
+ this.isUnmounted = true;
|
||||
+ this.disposeRecording();
|
||||
+ }
|
||||
+
|
||||
+ public onRecordStartEndClick = async (): Promise<void> => {
|
||||
+ if (this.state.isRecording) {
|
||||
+ this.stopRecording();
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ await this.startRecording();
|
||||
+ };
|
||||
+
|
||||
+ private startRecording = async (): Promise<void> => {
|
||||
+ if (this.isRequestingMedia || this.mediaRecorder) return;
|
||||
+
|
||||
+ if (!isVideoMessageRecorderSupported()) {
|
||||
+ if (!this.isUnmounted) {
|
||||
+ this.setState({ error: _t("composer|video_message_error") });
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ }
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ this.isRequestingMedia = true;
|
||||
+ this.shouldDiscardRecording = false;
|
||||
+ this.revokePreviewUrl();
|
||||
+ this.chunks = [];
|
||||
+
|
||||
+ try {
|
||||
+ const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
|
||||
+ if (this.isUnmounted) {
|
||||
+ stream.getTracks().forEach((track) => track.stop());
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ this.stream = stream;
|
||||
+ const mediaRecorder = new MediaRecorder(stream);
|
||||
+
|
||||
+ mediaRecorder.ondataavailable = (ev: BlobEvent): void => {
|
||||
+ if (ev.data.size > 0) {
|
||||
+ this.chunks.push(ev.data);
|
||||
+ }
|
||||
+ };
|
||||
+ mediaRecorder.onerror = (ev: Event): void => {
|
||||
+ logger.error("Error recording video message:", ev);
|
||||
+ this.setState({ error: _t("composer|video_message_error") });
|
||||
+ };
|
||||
+ mediaRecorder.onstop = this.onRecorderStop;
|
||||
+
|
||||
+ this.mediaRecorder = mediaRecorder;
|
||||
+ mediaRecorder.start();
|
||||
+
|
||||
+ this.setState({ error: undefined, isRecording: true, previewUrl: undefined });
|
||||
+ this.props.onRecordingStateChange(true);
|
||||
+ } catch (e) {
|
||||
+ logger.error("Error starting video message recording:", e);
|
||||
+ this.stopStream();
|
||||
+ if (!this.isUnmounted) {
|
||||
+ this.setState({ error: _t("composer|video_message_error"), isRecording: false, previewUrl: undefined });
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ }
|
||||
+ } finally {
|
||||
+ this.isRequestingMedia = false;
|
||||
+ }
|
||||
+ };
|
||||
+
|
||||
+ private stopRecording = (): void => {
|
||||
+ if (!this.mediaRecorder) return;
|
||||
+
|
||||
+ if (this.mediaRecorder.state === "inactive") {
|
||||
+ this.onRecorderStop();
|
||||
+ } else {
|
||||
+ this.mediaRecorder.stop();
|
||||
+ }
|
||||
+ };
|
||||
+
|
||||
+ private onRecorderStop = (): void => {
|
||||
+ const shouldDiscardRecording = this.shouldDiscardRecording;
|
||||
+ const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
|
||||
+
|
||||
+ this.mediaRecorder = undefined;
|
||||
+ this.stopStream();
|
||||
+
|
||||
+ if (this.isUnmounted) {
|
||||
+ this.chunks = [];
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ if (!hasRecording) {
|
||||
+ this.chunks = [];
|
||||
+ this.setState({ isRecording: false, previewUrl: undefined });
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "video/webm" });
|
||||
+ const previewUrl = URL.createObjectURL(blob);
|
||||
+
|
||||
+ this.chunks = [];
|
||||
+ this.revokePreviewUrl();
|
||||
+ this.setState({ isRecording: false, previewUrl });
|
||||
+ this.props.onRecordingStateChange(true);
|
||||
+ };
|
||||
+
|
||||
+ private onCancel = (): void => {
|
||||
+ this.shouldDiscardRecording = true;
|
||||
+ this.disposeRecording();
|
||||
+ this.setState({ error: undefined, isRecording: false, previewUrl: undefined });
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ };
|
||||
+
|
||||
+ private disposeRecording(): void {
|
||||
+ this.chunks = [];
|
||||
+ this.stopStream();
|
||||
+ this.revokePreviewUrl();
|
||||
+
|
||||
+ if (this.mediaRecorder) {
|
||||
+ this.mediaRecorder.ondataavailable = null;
|
||||
+ this.mediaRecorder.onerror = null;
|
||||
+ this.mediaRecorder.onstop = null;
|
||||
+
|
||||
+ if (this.mediaRecorder.state !== "inactive") {
|
||||
+ this.mediaRecorder.stop();
|
||||
+ }
|
||||
+ this.mediaRecorder = undefined;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ private stopStream(): void {
|
||||
+ this.stream?.getTracks().forEach((track) => track.stop());
|
||||
+ this.stream = undefined;
|
||||
+ }
|
||||
+
|
||||
+ private revokePreviewUrl(): void {
|
||||
+ if (this.state.previewUrl) {
|
||||
+ URL.revokeObjectURL(this.state.previewUrl);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ public render(): ReactNode {
|
||||
+ if (!this.state.isRecording && !this.state.previewUrl && !this.state.error) return null;
|
||||
+
|
||||
+ const stopButton = this.state.isRecording ? (
|
||||
+ <AccessibleButton
|
||||
+ className="mx_VoiceRecordComposerTile_stop"
|
||||
+ data-testid="video-message-stop-button"
|
||||
+ onClick={this.onRecordStartEndClick}
|
||||
+ title={_t("composer|stop_video_message")}
|
||||
+ >
|
||||
+ <StopSolidIcon />
|
||||
+ </AccessibleButton>
|
||||
+ ) : null;
|
||||
+
|
||||
+ const sendButton = this.state.previewUrl ? (
|
||||
+ <AccessibleButton
|
||||
+ className="mx_VoiceRecordComposerTile_stop"
|
||||
+ data-testid="video-message-send-button"
|
||||
+ disabled={true}
|
||||
+ onClick={null}
|
||||
+ title={_t("composer|send_video_message")}
|
||||
+ >
|
||||
+ <SendSolidIcon />
|
||||
+ </AccessibleButton>
|
||||
+ ) : null;
|
||||
+
|
||||
+ return (
|
||||
+ <>
|
||||
+ {this.state.error && (
|
||||
+ <span className="mx_VoiceRecordComposerTile_failedState" data-testid="video-message-error">
|
||||
+ {this.state.error}
|
||||
+ </span>
|
||||
+ )}
|
||||
+ <AccessibleButton
|
||||
+ className="mx_VoiceRecordComposerTile_delete"
|
||||
+ data-testid="video-message-cancel-button"
|
||||
+ onClick={this.onCancel}
|
||||
+ title={_t("composer|video_message_cancel")}
|
||||
+ >
|
||||
+ <DeleteIcon />
|
||||
+ </AccessibleButton>
|
||||
+ {stopButton}
|
||||
+ {sendButton}
|
||||
+ {this.state.previewUrl && (
|
||||
+ <video
|
||||
+ aria-label={_t("composer|video_message_preview")}
|
||||
+ className="mx_VoiceMessagePrimaryContainer"
|
||||
+ controls={true}
|
||||
+ data-testid="video-message-preview"
|
||||
+ src={this.state.previewUrl}
|
||||
+ />
|
||||
+ )}
|
||||
+ </>
|
||||
+ );
|
||||
+ }
|
||||
+}
|
||||
diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json
|
||||
index 4ed51db..e11fe90 100644
|
||||
--- a/apps/web/src/i18n/strings/en_EN.json
|
||||
+++ b/apps/web/src/i18n/strings/en_EN.json
|
||||
@@ -641,8 +641,14 @@
|
||||
"room_upgraded_notice": "This room has been replaced and is no longer active.",
|
||||
"send_button_title": "Send message",
|
||||
"send_button_voice_message": "Send voice message",
|
||||
+ "send_video_message": "Send video message",
|
||||
"send_voice_message": "Send voice message",
|
||||
+ "stop_video_message": "Stop video recording",
|
||||
"stop_voice_message": "Stop recording",
|
||||
+ "video_message_button": "Record video message",
|
||||
+ "video_message_cancel": "Cancel video message",
|
||||
+ "video_message_error": "Unable to record video message",
|
||||
+ "video_message_preview": "Video message preview",
|
||||
"voice_message_button": "Voice Message"
|
||||
},
|
||||
"console_dev_note": "If you know what you're doing, Element is open-source, be sure to check out our GitHub (https://github.com/vector-im/element-web/) and contribute!",
|
||||
@@ -0,0 +1,201 @@
|
||||
diff --git a/apps/web/src/components/views/rooms/MessageComposer.tsx b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||
index eea76b7..3483c28 100644
|
||||
--- a/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||
+++ b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||
@@ -629,6 +629,9 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||
key="controls_video_record"
|
||||
ref={this.videoRecordingButton}
|
||||
onRecordingStateChange={this.onVideoRecordingStateChange}
|
||||
+ relation={this.props.relation}
|
||||
+ replyToEvent={this.props.replyToEvent}
|
||||
+ room={this.props.room}
|
||||
/>,
|
||||
);
|
||||
} else if (this.context.tombstone) {
|
||||
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
index c5f0adc..851c64e 100644
|
||||
--- a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
@@ -6,22 +6,39 @@ Please see LICENSE files in the repository root for full details.
|
||||
*/
|
||||
|
||||
import React, { type ReactNode } from "react";
|
||||
+import { type IEventRelation, type MatrixEvent, type Room } from "matrix-js-sdk/src/matrix";
|
||||
import { logger } from "matrix-js-sdk/src/logger";
|
||||
import { DeleteIcon, SendSolidIcon, StopSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
|
||||
import { _t } from "../../../languageHandler";
|
||||
+import { MatrixClientPeg } from "../../../MatrixClientPeg";
|
||||
+import ContentMessages from "../../../ContentMessages";
|
||||
import AccessibleButton from "../elements/AccessibleButton";
|
||||
|
||||
interface IProps {
|
||||
onRecordingStateChange: (haveVideoRecording: boolean) => void;
|
||||
+ relation?: IEventRelation;
|
||||
+ replyToEvent?: MatrixEvent;
|
||||
+ room: Room;
|
||||
}
|
||||
|
||||
interface IState {
|
||||
error?: string;
|
||||
isRecording: boolean;
|
||||
+ isSending: boolean;
|
||||
previewUrl?: string;
|
||||
}
|
||||
|
||||
+const PREFERRED_VIDEO_MIME_TYPES = ["video/webm;codecs=vp8,opus", "video/webm"];
|
||||
+
|
||||
+function preferredVideoMimeType(): string | undefined {
|
||||
+ for (const mimeType of PREFERRED_VIDEO_MIME_TYPES) {
|
||||
+ if (MediaRecorder.isTypeSupported(mimeType)) {
|
||||
+ return mimeType;
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
export function isVideoMessageRecorderSupported(): boolean {
|
||||
return (
|
||||
typeof window !== "undefined" &&
|
||||
@@ -37,6 +54,7 @@ export function isVideoMessageRecorderSupported(): boolean {
|
||||
export default class VideoRecordComposerTile extends React.PureComponent<IProps, IState> {
|
||||
private chunks: Blob[] = [];
|
||||
private mediaRecorder?: MediaRecorder;
|
||||
+ private recordedFile?: File;
|
||||
private stream?: MediaStream;
|
||||
private isRequestingMedia = false;
|
||||
private isUnmounted = false;
|
||||
@@ -47,6 +65,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
|
||||
this.state = {
|
||||
isRecording: false,
|
||||
+ isSending: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -78,6 +97,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
this.isRequestingMedia = true;
|
||||
this.shouldDiscardRecording = false;
|
||||
this.revokePreviewUrl();
|
||||
+ this.recordedFile = undefined;
|
||||
this.chunks = [];
|
||||
|
||||
try {
|
||||
@@ -88,7 +108,8 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
}
|
||||
|
||||
this.stream = stream;
|
||||
- const mediaRecorder = new MediaRecorder(stream);
|
||||
+ const mimeType = preferredVideoMimeType();
|
||||
+ const mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
|
||||
|
||||
mediaRecorder.ondataavailable = (ev: BlobEvent): void => {
|
||||
if (ev.data.size > 0) {
|
||||
@@ -104,13 +125,18 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
this.mediaRecorder = mediaRecorder;
|
||||
mediaRecorder.start();
|
||||
|
||||
- this.setState({ error: undefined, isRecording: true, previewUrl: undefined });
|
||||
+ this.setState({ error: undefined, isRecording: true, isSending: false, previewUrl: undefined });
|
||||
this.props.onRecordingStateChange(true);
|
||||
} catch (e) {
|
||||
logger.error("Error starting video message recording:", e);
|
||||
this.stopStream();
|
||||
if (!this.isUnmounted) {
|
||||
- this.setState({ error: _t("composer|video_message_error"), isRecording: false, previewUrl: undefined });
|
||||
+ this.setState({
|
||||
+ error: _t("composer|video_message_error"),
|
||||
+ isRecording: false,
|
||||
+ isSending: false,
|
||||
+ previewUrl: undefined,
|
||||
+ });
|
||||
this.props.onRecordingStateChange(false);
|
||||
}
|
||||
} finally {
|
||||
@@ -129,6 +155,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
};
|
||||
|
||||
private onRecorderStop = (): void => {
|
||||
+ const mimeType = this.mediaRecorder?.mimeType;
|
||||
const shouldDiscardRecording = this.shouldDiscardRecording;
|
||||
const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
|
||||
|
||||
@@ -142,29 +169,63 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
|
||||
if (!hasRecording) {
|
||||
this.chunks = [];
|
||||
- this.setState({ isRecording: false, previewUrl: undefined });
|
||||
+ this.recordedFile = undefined;
|
||||
+ this.setState({ isRecording: false, isSending: false, previewUrl: undefined });
|
||||
this.props.onRecordingStateChange(false);
|
||||
return;
|
||||
}
|
||||
|
||||
- const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "video/webm" });
|
||||
- const previewUrl = URL.createObjectURL(blob);
|
||||
+ const blob = new Blob(this.chunks, { type: mimeType || this.chunks[0]?.type || "video/webm" });
|
||||
+ const file = new File([blob], `video-message-${Date.now()}.webm`, { type: blob.type || "video/webm" });
|
||||
+ const previewUrl = URL.createObjectURL(file);
|
||||
|
||||
this.chunks = [];
|
||||
+ this.recordedFile = file;
|
||||
this.revokePreviewUrl();
|
||||
- this.setState({ isRecording: false, previewUrl });
|
||||
+ this.setState({ isRecording: false, isSending: false, previewUrl });
|
||||
this.props.onRecordingStateChange(true);
|
||||
};
|
||||
|
||||
+ private onSend = async (): Promise<void> => {
|
||||
+ if (!this.recordedFile || this.state.isSending) return;
|
||||
+
|
||||
+ this.setState({ isSending: true });
|
||||
+
|
||||
+ try {
|
||||
+ await ContentMessages.sharedInstance().sendContentToRoom(
|
||||
+ this.recordedFile,
|
||||
+ this.props.room.roomId,
|
||||
+ this.props.relation,
|
||||
+ MatrixClientPeg.safeGet(),
|
||||
+ this.props.replyToEvent,
|
||||
+ );
|
||||
+ this.clearRecording();
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ } catch (e) {
|
||||
+ logger.error("Error sending video message recording:", e);
|
||||
+ if (!this.isUnmounted) {
|
||||
+ this.setState({ error: _t("composer|video_message_error"), isSending: false });
|
||||
+ }
|
||||
+ }
|
||||
+ };
|
||||
+
|
||||
private onCancel = (): void => {
|
||||
this.shouldDiscardRecording = true;
|
||||
this.disposeRecording();
|
||||
- this.setState({ error: undefined, isRecording: false, previewUrl: undefined });
|
||||
+ this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
|
||||
this.props.onRecordingStateChange(false);
|
||||
};
|
||||
|
||||
+ private clearRecording(): void {
|
||||
+ this.chunks = [];
|
||||
+ this.recordedFile = undefined;
|
||||
+ this.revokePreviewUrl();
|
||||
+ this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
|
||||
+ }
|
||||
+
|
||||
private disposeRecording(): void {
|
||||
this.chunks = [];
|
||||
+ this.recordedFile = undefined;
|
||||
this.stopStream();
|
||||
this.revokePreviewUrl();
|
||||
|
||||
@@ -209,8 +270,8 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
<AccessibleButton
|
||||
className="mx_VoiceRecordComposerTile_stop"
|
||||
data-testid="video-message-send-button"
|
||||
- disabled={true}
|
||||
- onClick={null}
|
||||
+ disabled={!this.recordedFile || this.state.isSending}
|
||||
+ onClick={this.onSend}
|
||||
title={_t("composer|send_video_message")}
|
||||
>
|
||||
<SendSolidIcon />
|
||||
@@ -0,0 +1,195 @@
|
||||
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
index 851c64e..4285c72 100644
|
||||
--- a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||
@@ -30,6 +30,7 @@ interface IState {
|
||||
}
|
||||
|
||||
const PREFERRED_VIDEO_MIME_TYPES = ["video/webm;codecs=vp8,opus", "video/webm"];
|
||||
+const MAX_VIDEO_RECORDING_MS = 120000;
|
||||
|
||||
function preferredVideoMimeType(): string | undefined {
|
||||
for (const mimeType of PREFERRED_VIDEO_MIME_TYPES) {
|
||||
@@ -39,6 +40,18 @@ function preferredVideoMimeType(): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
+function isPermissionDeniedError(error: unknown): boolean {
|
||||
+ if (error instanceof DOMException) {
|
||||
+ return error.name === "NotAllowedError" || error.name === "PermissionDeniedError";
|
||||
+ }
|
||||
+
|
||||
+ if (error instanceof Error) {
|
||||
+ return /permission denied|not allowed|denied permission/i.test(error.message);
|
||||
+ }
|
||||
+
|
||||
+ return false;
|
||||
+}
|
||||
+
|
||||
export function isVideoMessageRecorderSupported(): boolean {
|
||||
return (
|
||||
typeof window !== "undefined" &&
|
||||
@@ -56,6 +69,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
private mediaRecorder?: MediaRecorder;
|
||||
private recordedFile?: File;
|
||||
private stream?: MediaStream;
|
||||
+ private durationCapTimer?: number;
|
||||
private isRequestingMedia = false;
|
||||
private isUnmounted = false;
|
||||
private shouldDiscardRecording = false;
|
||||
@@ -72,6 +86,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
public componentWillUnmount(): void {
|
||||
this.isUnmounted = true;
|
||||
this.disposeRecording();
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
}
|
||||
|
||||
public onRecordStartEndClick = async (): Promise<void> => {
|
||||
@@ -96,6 +111,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
|
||||
this.isRequestingMedia = true;
|
||||
this.shouldDiscardRecording = false;
|
||||
+ this.clearDurationCapTimer();
|
||||
this.revokePreviewUrl();
|
||||
this.recordedFile = undefined;
|
||||
this.chunks = [];
|
||||
@@ -108,6 +124,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
}
|
||||
|
||||
this.stream = stream;
|
||||
+ this.bindStreamEndedHandlers(stream);
|
||||
const mimeType = preferredVideoMimeType();
|
||||
const mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
|
||||
|
||||
@@ -118,12 +135,13 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
};
|
||||
mediaRecorder.onerror = (ev: Event): void => {
|
||||
logger.error("Error recording video message:", ev);
|
||||
- this.setState({ error: _t("composer|video_message_error") });
|
||||
+ this.failRecording(_t("composer|video_message_error"));
|
||||
};
|
||||
mediaRecorder.onstop = this.onRecorderStop;
|
||||
|
||||
this.mediaRecorder = mediaRecorder;
|
||||
mediaRecorder.start();
|
||||
+ this.durationCapTimer = window.setTimeout(this.onDurationCap, MAX_VIDEO_RECORDING_MS);
|
||||
|
||||
this.setState({ error: undefined, isRecording: true, isSending: false, previewUrl: undefined });
|
||||
this.props.onRecordingStateChange(true);
|
||||
@@ -132,7 +150,9 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
this.stopStream();
|
||||
if (!this.isUnmounted) {
|
||||
this.setState({
|
||||
- error: _t("composer|video_message_error"),
|
||||
+ error: isPermissionDeniedError(e)
|
||||
+ ? _t("composer|video_message_permission_denied")
|
||||
+ : _t("composer|video_message_error"),
|
||||
isRecording: false,
|
||||
isSending: false,
|
||||
previewUrl: undefined,
|
||||
@@ -144,6 +164,21 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
}
|
||||
};
|
||||
|
||||
+ private onDurationCap = (): void => {
|
||||
+ this.durationCapTimer = undefined;
|
||||
+
|
||||
+ if (!this.mediaRecorder || this.mediaRecorder.state === "inactive") return;
|
||||
+
|
||||
+ this.stopRecording();
|
||||
+ };
|
||||
+
|
||||
+ private onStreamTrackEnded = (): void => {
|
||||
+ if (!this.mediaRecorder || this.mediaRecorder.state === "inactive") return;
|
||||
+
|
||||
+ logger.warn("Video message media stream ended before recording stopped");
|
||||
+ this.failRecording(_t("composer|video_message_error"));
|
||||
+ };
|
||||
+
|
||||
private stopRecording = (): void => {
|
||||
if (!this.mediaRecorder) return;
|
||||
|
||||
@@ -160,6 +195,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
|
||||
|
||||
this.mediaRecorder = undefined;
|
||||
+ this.clearDurationCapTimer();
|
||||
this.stopStream();
|
||||
|
||||
if (this.isUnmounted) {
|
||||
@@ -200,7 +236,9 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
this.props.replyToEvent,
|
||||
);
|
||||
this.clearRecording();
|
||||
- this.props.onRecordingStateChange(false);
|
||||
+ if (!this.isUnmounted) {
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ }
|
||||
} catch (e) {
|
||||
logger.error("Error sending video message recording:", e);
|
||||
if (!this.isUnmounted) {
|
||||
@@ -219,13 +257,18 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
private clearRecording(): void {
|
||||
this.chunks = [];
|
||||
this.recordedFile = undefined;
|
||||
+ this.clearDurationCapTimer();
|
||||
+ this.stopStream();
|
||||
this.revokePreviewUrl();
|
||||
+ if (this.isUnmounted) return;
|
||||
+
|
||||
this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
|
||||
}
|
||||
|
||||
private disposeRecording(): void {
|
||||
this.chunks = [];
|
||||
this.recordedFile = undefined;
|
||||
+ this.clearDurationCapTimer();
|
||||
this.stopStream();
|
||||
this.revokePreviewUrl();
|
||||
|
||||
@@ -241,8 +284,32 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||
}
|
||||
}
|
||||
|
||||
+ private failRecording(error: string): void {
|
||||
+ this.shouldDiscardRecording = true;
|
||||
+ this.disposeRecording();
|
||||
+
|
||||
+ if (this.isUnmounted) return;
|
||||
+
|
||||
+ this.setState({ error, isRecording: false, isSending: false, previewUrl: undefined });
|
||||
+ this.props.onRecordingStateChange(false);
|
||||
+ }
|
||||
+
|
||||
+ private clearDurationCapTimer(): void {
|
||||
+ if (this.durationCapTimer !== undefined) {
|
||||
+ window.clearTimeout(this.durationCapTimer);
|
||||
+ this.durationCapTimer = undefined;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ private bindStreamEndedHandlers(stream: MediaStream): void {
|
||||
+ stream.getTracks().forEach((track) => track.addEventListener("ended", this.onStreamTrackEnded));
|
||||
+ }
|
||||
+
|
||||
private stopStream(): void {
|
||||
- this.stream?.getTracks().forEach((track) => track.stop());
|
||||
+ this.stream?.getTracks().forEach((track) => {
|
||||
+ track.removeEventListener("ended", this.onStreamTrackEnded);
|
||||
+ track.stop();
|
||||
+ });
|
||||
this.stream = undefined;
|
||||
}
|
||||
|
||||
diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json
|
||||
index e11fe90..83b7c21 100644
|
||||
--- a/apps/web/src/i18n/strings/en_EN.json
|
||||
+++ b/apps/web/src/i18n/strings/en_EN.json
|
||||
@@ -648,6 +648,7 @@
|
||||
"video_message_button": "Record video message",
|
||||
"video_message_cancel": "Cancel video message",
|
||||
"video_message_error": "Unable to record video message",
|
||||
+ "video_message_permission_denied": "Camera permission denied",
|
||||
"video_message_preview": "Video message preview",
|
||||
"voice_message_button": "Voice Message"
|
||||
},
|
||||
@@ -0,0 +1,17 @@
|
||||
# Element Web patches
|
||||
|
||||
Put ordered local patch files in this directory as `*.patch`.
|
||||
|
||||
- Files are applied in lexical order.
|
||||
- Use zero-padded numeric prefixes when patch order matters, e.g. `0001-...patch`.
|
||||
- Keep non-patch files out of the order by using a different extension; `README.md` is ignored by the Nix filter.
|
||||
|
||||
## Regenerating a patch
|
||||
|
||||
Create a clean checkout of upstream Element Web, make the change under `apps/web`, then run from the repository root:
|
||||
|
||||
```sh
|
||||
git diff -- apps/web > package/element-web/patches/0001-description.patch
|
||||
```
|
||||
|
||||
Use one patch per logical change so the sequence stays reproducible and reviewable.
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
bash,
|
||||
cacert,
|
||||
coreutils,
|
||||
dockerTools,
|
||||
git,
|
||||
lib,
|
||||
nix,
|
||||
symlinkJoin,
|
||||
}:
|
||||
let
|
||||
name = "gitea-runner-nix-image";
|
||||
tag = "2026-06-07";
|
||||
root = symlinkJoin {
|
||||
name = "${name}-root";
|
||||
paths = [
|
||||
bash
|
||||
cacert
|
||||
coreutils
|
||||
git
|
||||
nix
|
||||
];
|
||||
};
|
||||
in
|
||||
dockerTools.buildLayeredImageWithNixDb {
|
||||
inherit name tag;
|
||||
|
||||
contents = [ root ];
|
||||
|
||||
fakeRootCommands = ''
|
||||
mkdir -p ./etc
|
||||
cat > ./etc/passwd <<'EOF'
|
||||
root:x:0:0:root:/root:/bin/bash
|
||||
nobody:x:65534:65534:nobody:/var/empty:/bin/false
|
||||
EOF
|
||||
cat > ./etc/group <<'EOF'
|
||||
root:x:0:
|
||||
nixbld:x:30000:
|
||||
nobody:x:65534:
|
||||
EOF
|
||||
for n in $(seq 1 10); do
|
||||
echo "nixbld$n:x:$((30000 + n)):30000:Nix build user $n:/var/empty:/bin/false" >> ./etc/passwd
|
||||
sed -i "s/^nixbld:x:30000:.*/&,nixbld$n/" ./etc/group
|
||||
done
|
||||
|
||||
mkdir -p ./nix/var/nix/{gcroots,profiles,temproots,userpool,db}
|
||||
mkdir -p ./nix/var/log/nix/drvs
|
||||
chmod 1777 ./nix/var/nix/gcroots ./nix/var/nix/profiles
|
||||
'';
|
||||
|
||||
extraCommands = ''
|
||||
mkdir -p etc/nix root tmp
|
||||
chmod 1777 tmp
|
||||
|
||||
cat > etc/nix/nix.conf <<'EOF'
|
||||
accept-flake-config = true
|
||||
experimental-features = nix-command flakes
|
||||
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||
trusted-users = root
|
||||
sandbox = false
|
||||
EOF
|
||||
'';
|
||||
|
||||
config = {
|
||||
Cmd = [ "${bash}/bin/bash" ];
|
||||
Env = [
|
||||
"HOME=/root"
|
||||
"PATH=${lib.makeBinPath [ bash coreutils git nix ]}"
|
||||
"SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||
"NIX_SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||
"USER=root"
|
||||
];
|
||||
Labels = {
|
||||
"org.opencontainers.image.description" = "Nix-capable Gitea Actions job image";
|
||||
"org.opencontainers.image.ref.name" = "${name}:${tag}";
|
||||
"org.opencontainers.image.source" = "https://gitea.hectic-lab.com/hectic-lab/util.nix";
|
||||
};
|
||||
WorkingDir = "/workspace";
|
||||
};
|
||||
}
|
||||
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
|
||||
Commands: []*cli.Command{
|
||||
newUserCommand(),
|
||||
newRepoSyncReleasesCommand(),
|
||||
newHeatmapCommand(),
|
||||
newRegenerateCommand(),
|
||||
newAuthCommand(),
|
||||
newSendMailCommand(),
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
repo_service "code.gitea.io/gitea/services/repository"
|
||||
|
||||
"github.com/urfave/cli/v3"
|
||||
)
|
||||
|
||||
func newHeatmapCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "heatmap",
|
||||
Usage: "Manage heatmap indexes",
|
||||
Commands: []*cli.Command{
|
||||
newHeatmapReindexCommand(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newHeatmapReindexCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "reindex",
|
||||
Usage: "Reindex heatmap contributions from repository default branches",
|
||||
Action: runHeatmapReindex,
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "all",
|
||||
Usage: "Reindex all repositories",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "repo",
|
||||
Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "owner",
|
||||
Usage: "Owner name for --repo",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
|
||||
if err := initDB(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := git.InitSimple(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reindexAll := c.Bool("all")
|
||||
repoName := c.String("repo")
|
||||
ownerName := c.String("owner")
|
||||
if reindexAll {
|
||||
if repoName != "" || ownerName != "" {
|
||||
return fmt.Errorf("--all cannot be combined with --repo or --owner")
|
||||
}
|
||||
return reindexAllHeatmapRepositories(ctx)
|
||||
}
|
||||
if repoName == "" {
|
||||
return fmt.Errorf("either --all or --repo must be provided")
|
||||
}
|
||||
|
||||
ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return reindexHeatmapRepository(ctx, repo)
|
||||
}
|
||||
|
||||
func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
|
||||
if ownerName != "" {
|
||||
if strings.Contains(repoName, "/") {
|
||||
return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
|
||||
}
|
||||
return ownerName, repoName, nil
|
||||
}
|
||||
|
||||
ownerName, repoName, ok := strings.Cut(repoName, "/")
|
||||
if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
|
||||
return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
|
||||
}
|
||||
return ownerName, repoName, nil
|
||||
}
|
||||
|
||||
func reindexAllHeatmapRepositories(ctx context.Context) error {
|
||||
for page := 1; ; page++ {
|
||||
repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
|
||||
ListOptions: db.ListOptions{
|
||||
PageSize: repo_model.RepositoryListDefaultPageSize,
|
||||
Page: page,
|
||||
},
|
||||
Private: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("SearchRepositoryByName: %w", err)
|
||||
}
|
||||
if len(repos) == 0 {
|
||||
break
|
||||
}
|
||||
log.Trace("Processing next %d repos of %d", len(repos), count)
|
||||
for _, repo := range repos {
|
||||
if err := reindexHeatmapRepository(ctx, repo); err != nil {
|
||||
log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
|
||||
log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
|
||||
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||
return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
repo_service "code.gitea.io/gitea/services/repository"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHeatmapAdminReindex(t *testing.T) {
|
||||
repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
|
||||
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||
})
|
||||
|
||||
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||
contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 1)
|
||||
assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
|
||||
|
||||
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
|
||||
newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
|
||||
{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||
})
|
||||
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 1)
|
||||
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
|
||||
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
|
||||
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 1)
|
||||
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
|
||||
}
|
||||
|
||||
func TestHeatmapAdminRepoSelector(t *testing.T) {
|
||||
owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "user", owner)
|
||||
assert.Equal(t, "repo", repo)
|
||||
|
||||
owner, repo, err = parseHeatmapRepoSelector("user", "repo")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "user", owner)
|
||||
assert.Equal(t, "repo", repo)
|
||||
|
||||
_, _, err = parseHeatmapRepoSelector("", "repo")
|
||||
assert.ErrorContains(t, err, "owner/name")
|
||||
_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
|
||||
assert.ErrorContains(t, err, "when --owner is set")
|
||||
}
|
||||
|
||||
type heatmapAdminTestCommit struct {
|
||||
Branch string
|
||||
Mark string
|
||||
Parent string
|
||||
AuthorName string
|
||||
AuthorEmail string
|
||||
CommitterName string
|
||||
CommitterEmail string
|
||||
AuthorDate string
|
||||
}
|
||||
|
||||
func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
|
||||
t.Helper()
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
|
||||
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||
repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
|
||||
require.NoError(t, err)
|
||||
|
||||
marks := runHeatmapAdminFastImport(t, repo, commits)
|
||||
repo.IsEmpty = false
|
||||
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
|
||||
return repo, marks
|
||||
}
|
||||
|
||||
func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
var stream strings.Builder
|
||||
branchTips := make(map[string]string)
|
||||
for i, commit := range commits {
|
||||
mark := fmt.Sprintf(":%d", i+1)
|
||||
branchRef := "refs/heads/" + commit.Branch
|
||||
stream.WriteString("commit " + branchRef + "\n")
|
||||
stream.WriteString("mark " + mark + "\n")
|
||||
stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
|
||||
stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
|
||||
message := "heatmap admin " + commit.Mark
|
||||
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
|
||||
if parentMark := branchTips[commit.Branch]; parentMark != "" {
|
||||
stream.WriteString("from " + parentMark + "\n")
|
||||
} else if commit.Parent != "" {
|
||||
stream.WriteString("from " + commit.Parent + "\n")
|
||||
}
|
||||
content := commit.Mark + "\n"
|
||||
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
|
||||
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
|
||||
branchTips[commit.Branch] = mark
|
||||
}
|
||||
|
||||
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
|
||||
WithDir(repo.RepoPath()).
|
||||
WithStdinCopy(strings.NewReader(stream.String())).
|
||||
Run(t.Context()))
|
||||
|
||||
commitSHAs := make(map[string]string, len(commits))
|
||||
for _, commit := range commits {
|
||||
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
|
||||
AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
|
||||
AddDynamicArguments("refs/heads/" + commit.Branch).
|
||||
WithDir(repo.RepoPath()).
|
||||
RunStdString(t.Context())
|
||||
require.NoError(t, err)
|
||||
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
|
||||
}
|
||||
return commitSHAs
|
||||
}
|
||||
|
||||
func heatmapAdminSignatureLine(kind, name, email, date string) string {
|
||||
parsed, err := time.Parse(time.RFC3339, date)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
|
||||
}
|
||||
|
||||
func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
|
||||
t.Helper()
|
||||
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
|
||||
require.NoError(t, err)
|
||||
return contributions
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package activities
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"xorm.io/xorm/schemas"
|
||||
)
|
||||
|
||||
// HeatmapContribution stores commit contributions counted for profile heatmaps.
|
||||
type HeatmapContribution struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
UserID int64 `xorm:"NOT NULL"`
|
||||
RepoID int64 `xorm:"NOT NULL"`
|
||||
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
|
||||
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
|
||||
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
|
||||
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
|
||||
}
|
||||
|
||||
// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing.
|
||||
type HeatmapContributionIdentity struct {
|
||||
RepoID int64
|
||||
CommitSHA string
|
||||
UserID int64
|
||||
}
|
||||
|
||||
func init() {
|
||||
db.RegisterModel(new(HeatmapContribution))
|
||||
}
|
||||
|
||||
// TableIndices implements xorm's TableIndices interface.
|
||||
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
|
||||
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
|
||||
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
|
||||
|
||||
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
|
||||
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
|
||||
|
||||
return []*schemas.Index{uniqueContribution, userAuthorRepo}
|
||||
}
|
||||
|
||||
// UpsertHeatmapContribution creates or updates a commit contribution without duplicating counted rows.
|
||||
func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContribution) error {
|
||||
return db.WithTx(ctx, func(ctx context.Context) error {
|
||||
e := db.GetEngine(ctx)
|
||||
|
||||
rows, err := e.Where("repo_id=? AND commit_sha=? AND user_id=?", contribution.RepoID, contribution.CommitSHA, contribution.UserID).
|
||||
Cols("author_email", "author_unix").
|
||||
Update(contribution)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
has, err := e.Exist(&HeatmapContribution{RepoID: contribution.RepoID, CommitSHA: contribution.CommitSHA, UserID: contribution.UserID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if has {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = e.Insert(contribution)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// CountHeatmapContributions returns counted heatmap contribution rows for a user.
|
||||
func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) {
|
||||
return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution))
|
||||
}
|
||||
|
||||
// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities.
|
||||
func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error {
|
||||
if len(identities) == 0 {
|
||||
_, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution))
|
||||
return err
|
||||
}
|
||||
|
||||
current := make(map[HeatmapContributionIdentity]struct{}, len(identities))
|
||||
for _, identity := range identities {
|
||||
current[identity] = struct{}{}
|
||||
}
|
||||
|
||||
contributions, err := FindHeatmapContributionsByRepo(ctx, repoID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, contribution := range contributions {
|
||||
identity := HeatmapContributionIdentity{
|
||||
RepoID: contribution.RepoID,
|
||||
CommitSHA: contribution.CommitSHA,
|
||||
UserID: contribution.UserID,
|
||||
}
|
||||
if _, ok := current[identity]; ok {
|
||||
continue
|
||||
}
|
||||
if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository.
|
||||
func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) {
|
||||
contributions := make([]*HeatmapContribution, 0)
|
||||
return contributions, db.GetEngine(ctx).
|
||||
Where("repo_id=?", repoID).
|
||||
OrderBy("author_unix ASC, commit_sha ASC, user_id ASC").
|
||||
Find(&contributions)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package activities_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHeatmapContributionModel(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
assert.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
|
||||
const (
|
||||
userID = int64(2)
|
||||
repoID = int64(1)
|
||||
commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
authorUnix = timeutil.TimeStamp(1579089600)
|
||||
)
|
||||
|
||||
contribution := &activities_model.HeatmapContribution{
|
||||
UserID: userID,
|
||||
RepoID: repoID,
|
||||
CommitSHA: commitSHA,
|
||||
AuthorEmail: "user2@example.com",
|
||||
AuthorUnix: authorUnix,
|
||||
}
|
||||
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), contribution))
|
||||
|
||||
count, err := activities_model.CountHeatmapContributions(t.Context(), userID)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1, count)
|
||||
|
||||
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||
UserID: userID,
|
||||
RepoID: repoID,
|
||||
CommitSHA: commitSHA,
|
||||
AuthorEmail: "changed@example.com",
|
||||
AuthorUnix: authorUnix + 900,
|
||||
}))
|
||||
|
||||
count, err = activities_model.CountHeatmapContributions(t.Context(), userID)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1, count)
|
||||
|
||||
stored := &activities_model.HeatmapContribution{UserID: userID, RepoID: repoID, CommitSHA: commitSHA}
|
||||
has, err := db.GetEngine(t.Context()).Get(stored)
|
||||
require.NoError(t, err)
|
||||
require.True(t, has)
|
||||
assert.Equal(t, "changed@example.com", stored.AuthorEmail)
|
||||
assert.Equal(t, authorUnix+900, stored.AuthorUnix)
|
||||
assert.NotZero(t, stored.CreatedUnix)
|
||||
assert.NotZero(t, stored.UpdatedUnix)
|
||||
|
||||
err = db.Insert(t.Context(), &activities_model.HeatmapContribution{
|
||||
UserID: userID,
|
||||
RepoID: repoID,
|
||||
CommitSHA: commitSHA,
|
||||
AuthorEmail: "duplicate@example.com",
|
||||
AuthorUnix: authorUnix,
|
||||
})
|
||||
assert.Error(t, err)
|
||||
|
||||
includePrivate, err := user_model.GetIncludePrivateContributions(t.Context(), userID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, includePrivate)
|
||||
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), userID, true))
|
||||
includePrivate, err = user_model.GetIncludePrivateContributions(t.Context(), userID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, includePrivate)
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"code.gitea.io/gitea/models/organization"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/structs"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
)
|
||||
|
||||
@@ -38,35 +39,45 @@ func getUserHeatmapData(ctx context.Context, user *user_model.User, team *organi
|
||||
|
||||
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
|
||||
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
|
||||
groupBy := "created_unix / 900 * 900"
|
||||
groupBy := "author_unix / 900 * 900"
|
||||
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
|
||||
switch {
|
||||
case setting.Database.Type.IsMySQL():
|
||||
groupBy = "created_unix DIV 900 * 900"
|
||||
groupBy = "author_unix DIV 900 * 900"
|
||||
case setting.Database.Type.IsMSSQL():
|
||||
groupByName = groupBy
|
||||
}
|
||||
|
||||
cond, err := ActivityQueryCondition(ctx, GetFeedsOptions{
|
||||
RequestedUser: user,
|
||||
RequestedTeam: team,
|
||||
Actor: doer,
|
||||
IncludePrivate: true, // don't filter by private, as we already filter by repo access
|
||||
IncludeDeleted: true,
|
||||
// * Heatmaps for individual users only include actions that the user themself did.
|
||||
// * For organizations actions by all users that were made in owned
|
||||
// repositories are counted.
|
||||
OnlyPerformedBy: !user.IsOrganization(),
|
||||
})
|
||||
includePrivate, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return hdata, db.GetEngine(ctx).
|
||||
Select(groupBy+" AS timestamp, count(user_id) as contributions").
|
||||
Table("action").
|
||||
Where(cond).
|
||||
And("created_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
|
||||
sess := db.GetEngine(ctx).
|
||||
Select(groupBy+" AS timestamp, count(heatmap_contribution.user_id) as contributions").
|
||||
Table("heatmap_contribution").
|
||||
Join("INNER", "repository", "repository.id = heatmap_contribution.repo_id").
|
||||
Join("INNER", "`user` AS repo_owner", "repo_owner.id = repository.owner_id").
|
||||
Where("author_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
|
||||
And("author_unix <= ?", timeutil.TimeStampNow())
|
||||
|
||||
if !includePrivate {
|
||||
sess = sess.And("repository.is_private = ?", false).
|
||||
And("repo_owner.visibility = ?", structs.VisibleTypePublic)
|
||||
}
|
||||
|
||||
if user.IsOrganization() {
|
||||
sess = sess.And("repository.owner_id = ?", user.ID)
|
||||
if team != nil && !team.IncludesAllRepositories {
|
||||
sess = sess.Join("INNER", "team_repo", "team_repo.repo_id = repository.id").
|
||||
And("team_repo.org_id = ?", team.OrgID).
|
||||
And("team_repo.team_id = ?", team.ID)
|
||||
}
|
||||
} else {
|
||||
sess = sess.And("heatmap_contribution.user_id = ?", user.ID)
|
||||
}
|
||||
|
||||
return hdata, sess.
|
||||
GroupBy(groupByName).
|
||||
OrderBy("timestamp").
|
||||
Find(&hdata)
|
||||
|
||||
@@ -4,19 +4,43 @@
|
||||
package activities_test
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/models/organization"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/structs"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetUserHeatmapDataByUser(t *testing.T) {
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||
|
||||
// Mock time
|
||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
insertHeatmapContribution(t, 2, 1, "user2-public-author-date", 1603009800)
|
||||
insertHeatmapContribution(t, 2, 1, "user2-public-same-bucket", 1603009850)
|
||||
insertHeatmapContribution(t, 2, 2, "user2-private-hidden", 1603010700)
|
||||
insertHeatmapContribution(t, 2, 4, "user2-hidden-owner-hidden", 1603010700)
|
||||
insertHeatmapContribution(t, 2, 1, "user2-future-hidden", timeutil.TimeStampNow()+900)
|
||||
insertHeatmapContribution(t, 2, 999999, "user2-deleted-repo-hidden", 1603011600)
|
||||
insertHeatmapContribution(t, 3, 1, "other-author-ignored", 1603010700)
|
||||
setUserVisibility(t, 5, structs.VisibleTypePrivate)
|
||||
|
||||
testCases := []struct {
|
||||
desc string
|
||||
userID int64
|
||||
@@ -25,42 +49,29 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
|
||||
JSONResult string
|
||||
}{
|
||||
{
|
||||
"self looks at action in private repo",
|
||||
2, 2, 1, `[{"timestamp":1603227600,"contributions":1}]`,
|
||||
"self sees public author-date contributions only",
|
||||
2, 2, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||
},
|
||||
{
|
||||
"admin looks at action in private repo",
|
||||
2, 1, 1, `[{"timestamp":1603227600,"contributions":1}]`,
|
||||
"admin sees public author-date contributions only",
|
||||
2, 1, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||
},
|
||||
{
|
||||
"other user looks at action in private repo",
|
||||
2, 3, 0, `[]`,
|
||||
"other user sees public author-date contributions only",
|
||||
2, 3, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||
},
|
||||
{
|
||||
"nobody looks at action in private repo",
|
||||
2, 0, 0, `[]`,
|
||||
"anonymous sees public author-date contributions only",
|
||||
2, 0, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||
},
|
||||
{
|
||||
"collaborator looks at action in private repo",
|
||||
16, 15, 1, `[{"timestamp":1603267200,"contributions":1}]`,
|
||||
},
|
||||
{
|
||||
"no action action not performed by target user",
|
||||
"different author is not counted for target user",
|
||||
3, 3, 0, `[]`,
|
||||
},
|
||||
{
|
||||
"multiple actions performed with two grouped together",
|
||||
10, 10, 3, `[{"timestamp":1603009800,"contributions":1},{"timestamp":1603010700,"contributions":2}]`,
|
||||
},
|
||||
}
|
||||
// Prepare
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
// Mock time
|
||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
|
||||
|
||||
var doer *user_model.User
|
||||
@@ -68,30 +79,181 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
|
||||
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
|
||||
}
|
||||
|
||||
// get the action for comparison
|
||||
actions, count, err := activities_model.GetFeeds(t.Context(), activities_model.GetFeedsOptions{
|
||||
RequestedUser: user,
|
||||
Actor: doer,
|
||||
IncludePrivate: true,
|
||||
OnlyPerformedBy: true,
|
||||
IncludeDeleted: true,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Get the heatmap and compare
|
||||
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.CountResult, countHeatmapContributions(heatmap), "testcase '%s'", tc.desc)
|
||||
|
||||
// Test JSON rendering
|
||||
jsonData, err := json.Marshal(heatmap)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, tc.JSONResult, string(jsonData))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetUserHeatmapDataByUserUsesCurrentRepoVisibility(t *testing.T) {
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||
|
||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
insertHeatmapContribution(t, 2, 1, "user2-visibility-flip", 1603009800)
|
||||
|
||||
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||
require.NoError(t, err)
|
||||
assertHeatmapJSON(t, heatmap, `[{
|
||||
"timestamp": 1603009800,
|
||||
"contributions": 1
|
||||
}]`)
|
||||
|
||||
setRepoPrivate(t, 1, true)
|
||||
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, heatmap)
|
||||
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, true))
|
||||
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||
require.NoError(t, err)
|
||||
assertHeatmapJSON(t, heatmap, `[{
|
||||
"timestamp": 1603009800,
|
||||
"contributions": 1
|
||||
}]`)
|
||||
}
|
||||
|
||||
func TestGetUserHeatmapDataByOrgTeam(t *testing.T) {
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 3, false))
|
||||
|
||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
insertHeatmapContribution(t, 2, 32, "org-team-public", 1603009800)
|
||||
insertHeatmapContribution(t, 2, 3, "org-team-private-hidden", 1603010700)
|
||||
insertHeatmapContribution(t, 2, 1, "non-org-repo-ignored", 1603011600)
|
||||
|
||||
org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3})
|
||||
team := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: 7})
|
||||
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||
|
||||
heatmap, err := activities_model.GetUserHeatmapDataByOrgTeam(t.Context(), org, team, doer)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, countHeatmapContributions(heatmap))
|
||||
|
||||
jsonData, err := json.Marshal(heatmap)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `[{"timestamp":1603009800,"contributions":1}]`, string(jsonData))
|
||||
}
|
||||
|
||||
func TestUserHeatmapPrivateContributionsOptIn(t *testing.T) {
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
|
||||
|
||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
insertHeatmapContribution(t, 16, 21, "user16-public", 1603009800)
|
||||
insertHeatmapContribution(t, 16, 22, "user16-private-one", 1603009850)
|
||||
insertHeatmapContribution(t, 16, 22, "user16-private-two", 1603010700)
|
||||
|
||||
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 16})
|
||||
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||
authenticated := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3})
|
||||
collaborator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15})
|
||||
|
||||
for _, doer := range []*user_model.User{nil, target, admin, authenticated, collaborator} {
|
||||
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, doer)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, countHeatmapContributions(heatmap), "private contributions should be hidden by default from %s", heatmapDoerName(doer))
|
||||
}
|
||||
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, true))
|
||||
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||
require.NoError(t, err)
|
||||
assertHeatmapJSON(t, heatmap, `[{"timestamp":1603009800,"contributions":2},{"timestamp":1603010700,"contributions":1}]`)
|
||||
|
||||
target.KeepActivityPrivate = true
|
||||
_, err = db.GetEngine(t.Context()).ID(target.ID).Cols("keep_activity_private").Update(target)
|
||||
require.NoError(t, err)
|
||||
|
||||
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, heatmap)
|
||||
|
||||
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, authenticated)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, heatmap)
|
||||
|
||||
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, target)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, countHeatmapContributions(heatmap))
|
||||
|
||||
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, admin)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, countHeatmapContributions(heatmap))
|
||||
}
|
||||
|
||||
func insertHeatmapContribution(t *testing.T, userID, repoID int64, commitSHA string, authorUnix timeutil.TimeStamp) {
|
||||
t.Helper()
|
||||
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||
UserID: userID,
|
||||
RepoID: repoID,
|
||||
CommitSHA: fmt.Sprintf("%040x", sha1.Sum([]byte(commitSHA))),
|
||||
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
|
||||
AuthorUnix: authorUnix,
|
||||
}))
|
||||
}
|
||||
|
||||
func setRepoPrivate(t *testing.T, repoID int64, isPrivate bool) {
|
||||
t.Helper()
|
||||
repo := &repo_model.Repository{ID: repoID, IsPrivate: isPrivate}
|
||||
_, err := db.GetEngine(t.Context()).ID(repoID).Cols("is_private").Update(repo)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func setUserVisibility(t *testing.T, userID int64, visibility structs.VisibleType) {
|
||||
t.Helper()
|
||||
user := &user_model.User{ID: userID, Visibility: visibility}
|
||||
_, err := db.GetEngine(t.Context()).ID(userID).Cols("visibility").Update(user)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func assertHeatmapJSON(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected string) {
|
||||
t.Helper()
|
||||
jsonData, err := json.Marshal(heatmap)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, expected, string(jsonData))
|
||||
|
||||
var decoded []map[string]any
|
||||
require.NoError(t, json.Unmarshal(jsonData, &decoded))
|
||||
for _, entry := range decoded {
|
||||
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, mapKeys(entry))
|
||||
}
|
||||
}
|
||||
|
||||
func mapKeys[K comparable, V any](m map[K]V) []K {
|
||||
keys := make([]K, 0, len(m))
|
||||
for key := range m {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func countHeatmapContributions(heatmap []*activities_model.UserHeatmapData) int {
|
||||
var contributions int
|
||||
for _, hm := range heatmap {
|
||||
contributions += int(hm.Contributions)
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, actions, contributions, "invalid action count: did the test data became too old?")
|
||||
assert.Equal(t, count, int64(contributions))
|
||||
assert.Equal(t, tc.CountResult, contributions, "testcase '%s'", tc.desc)
|
||||
|
||||
// Test JSON rendering
|
||||
jsonData, err := json.Marshal(heatmap)
|
||||
assert.NoError(t, err)
|
||||
assert.JSONEq(t, tc.JSONResult, string(jsonData))
|
||||
}
|
||||
return contributions
|
||||
}
|
||||
|
||||
func heatmapDoerName(doer *user_model.User) string {
|
||||
if doer == nil {
|
||||
return "anonymous"
|
||||
}
|
||||
return doer.Name
|
||||
}
|
||||
|
||||
@@ -405,6 +405,7 @@ func prepareMigrationTasks() []*migration {
|
||||
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
|
||||
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
|
||||
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
|
||||
newMigration(331, "Create heatmap contribution table", v1_26.CreateHeatmapContributionTable),
|
||||
}
|
||||
return preparedMigrations
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package v1_26
|
||||
|
||||
import (
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"xorm.io/xorm"
|
||||
"xorm.io/xorm/schemas"
|
||||
)
|
||||
|
||||
type HeatmapContribution struct { //revive:disable-line:exported
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
UserID int64 `xorm:"NOT NULL"`
|
||||
RepoID int64 `xorm:"NOT NULL"`
|
||||
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
|
||||
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
|
||||
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
|
||||
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
|
||||
}
|
||||
|
||||
// TableIndices implements xorm's TableIndices interface.
|
||||
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
|
||||
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
|
||||
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
|
||||
|
||||
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
|
||||
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
|
||||
|
||||
return []*schemas.Index{uniqueContribution, userAuthorRepo}
|
||||
}
|
||||
|
||||
func CreateHeatmapContributionTable(x *xorm.Engine) error {
|
||||
_, err := x.SyncWithOptions(xorm.SyncOptions{IgnoreDropIndices: true}, new(HeatmapContribution))
|
||||
return err
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
@@ -19,6 +20,8 @@ import (
|
||||
"xorm.io/xorm/convert"
|
||||
)
|
||||
|
||||
const SettingsKeyIncludePrivateContributions = "include_private_contributions"
|
||||
|
||||
// Setting is a key value store of user settings
|
||||
type Setting struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
@@ -254,3 +257,17 @@ func SetUserSettingJSON[T any](ctx context.Context, userID int64, key string, va
|
||||
}
|
||||
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
|
||||
}
|
||||
|
||||
// GetIncludePrivateContributions returns whether a user opted in to private heatmap contributions.
|
||||
func GetIncludePrivateContributions(ctx context.Context, userID int64) (bool, error) {
|
||||
value, err := GetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, "false")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return strconv.ParseBool(value)
|
||||
}
|
||||
|
||||
// SetIncludePrivateContributions stores whether a user opted in to private heatmap contributions.
|
||||
func SetIncludePrivateContributions(ctx context.Context, userID int64, include bool) error {
|
||||
return SetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, strconv.FormatBool(include))
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ type UserSettings struct {
|
||||
// Privacy
|
||||
HideEmail bool `json:"hide_email"`
|
||||
HideActivity bool `json:"hide_activity"`
|
||||
IncludePrivateContributions bool `json:"include_private_contributions"`
|
||||
}
|
||||
|
||||
// UserSettingsOptions represents options to change user settings
|
||||
@@ -97,6 +98,7 @@ type UserSettingsOptions struct {
|
||||
// Privacy
|
||||
HideEmail *bool `json:"hide_email"`
|
||||
HideActivity *bool `json:"hide_activity"`
|
||||
IncludePrivateContributions *bool `json:"include_private_contributions"`
|
||||
}
|
||||
|
||||
// RenameUserOption options when renaming a user
|
||||
|
||||
@@ -708,6 +708,8 @@
|
||||
"settings.privacy": "Privacy",
|
||||
"settings.keep_activity_private": "Hide Activity from profile page",
|
||||
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
|
||||
"settings.include_private_contributions": "Show private contributions on profile heatmap",
|
||||
"settings.include_private_contributions_popup": "Publicly reveals only contribution dates and counts from non-public repositories, never repository or commit details.",
|
||||
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
|
||||
"settings.federated_avatar_lookup": "Federated Avatar Lookup",
|
||||
"settings.enable_custom_avatar": "Use Custom Avatar",
|
||||
|
||||
@@ -24,7 +24,12 @@ func GetUserSettings(ctx *context.APIContext) {
|
||||
// responses:
|
||||
// "200":
|
||||
// "$ref": "#/responses/UserSettings"
|
||||
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
|
||||
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
|
||||
if err != nil {
|
||||
ctx.APIErrorInternal(err)
|
||||
return
|
||||
}
|
||||
ctx.JSON(http.StatusOK, settings)
|
||||
}
|
||||
|
||||
// UpdateUserSettings returns user settings
|
||||
@@ -55,11 +60,17 @@ func UpdateUserSettings(ctx *context.APIContext) {
|
||||
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
|
||||
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
|
||||
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
|
||||
IncludePrivateContributions: optional.FromPtr(form.IncludePrivateContributions),
|
||||
}
|
||||
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
|
||||
ctx.APIErrorInternal(err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
|
||||
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
|
||||
if err != nil {
|
||||
ctx.APIErrorInternal(err)
|
||||
return
|
||||
}
|
||||
ctx.JSON(http.StatusOK, settings)
|
||||
}
|
||||
|
||||
@@ -48,16 +48,32 @@ func Profile(ctx *context.Context) {
|
||||
ctx.Data["PageIsSettingsProfile"] = true
|
||||
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
||||
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
||||
if !loadProfilePrivateContributionSetting(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
||||
}
|
||||
|
||||
func loadProfilePrivateContributionSetting(ctx *context.Context) bool {
|
||||
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, ctx.Doer.ID)
|
||||
if err != nil {
|
||||
ctx.ServerError("GetIncludePrivateContributions", err)
|
||||
return false
|
||||
}
|
||||
ctx.Data["IncludePrivateContributions"] = includePrivateContributions
|
||||
return true
|
||||
}
|
||||
|
||||
// ProfilePost response for change user's profile
|
||||
func ProfilePost(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("settings_title")
|
||||
ctx.Data["PageIsSettingsProfile"] = true
|
||||
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
||||
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
||||
if !loadProfilePrivateContributionSetting(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
if ctx.HasError() {
|
||||
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
||||
@@ -100,6 +116,7 @@ func ProfilePost(ctx *context.Context) {
|
||||
Location: optional.Some(form.Location),
|
||||
Visibility: optional.Some(form.Visibility),
|
||||
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
|
||||
IncludePrivateContributions: optional.Some(form.IncludePrivateContributions),
|
||||
}
|
||||
|
||||
if form.FullName != "" {
|
||||
|
||||
@@ -86,7 +86,11 @@ func toUser(ctx context.Context, user *user_model.User, signed, authed bool) *ap
|
||||
}
|
||||
|
||||
// User2UserSettings return UserSettings based on a user
|
||||
func User2UserSettings(user *user_model.User) api.UserSettings {
|
||||
func User2UserSettings(ctx context.Context, user *user_model.User) (api.UserSettings, error) {
|
||||
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
|
||||
if err != nil {
|
||||
return api.UserSettings{}, err
|
||||
}
|
||||
return api.UserSettings{
|
||||
FullName: user.FullName,
|
||||
Website: user.Website,
|
||||
@@ -96,8 +100,9 @@ func User2UserSettings(user *user_model.User) api.UserSettings {
|
||||
Theme: user.Theme,
|
||||
HideEmail: user.KeepEmailPrivate,
|
||||
HideActivity: user.KeepActivityPrivate,
|
||||
IncludePrivateContributions: includePrivateContributions,
|
||||
DiffViewStyle: user.DiffViewStyle,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ToUserAndPermission return User and its collaboration permission for a repository
|
||||
|
||||
@@ -216,6 +216,7 @@ type UpdateProfileForm struct {
|
||||
Description string `binding:"MaxSize(255)"`
|
||||
Visibility structs.VisibleType
|
||||
KeepActivityPrivate bool
|
||||
IncludePrivateContributions bool
|
||||
}
|
||||
|
||||
// Validate validates the fields
|
||||
|
||||
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
||||
defer gitRepo.Close()
|
||||
|
||||
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
|
||||
indexHeatmap := false
|
||||
if len(results) > 0 {
|
||||
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
|
||||
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
|
||||
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
||||
if result.RefName.IsPull() {
|
||||
continue
|
||||
}
|
||||
if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
|
||||
indexHeatmap = true
|
||||
}
|
||||
|
||||
// Create reference
|
||||
if result.OldCommitID == "" {
|
||||
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
||||
NewCommitID: newCommitID,
|
||||
}, theCommits)
|
||||
}
|
||||
if indexHeatmap {
|
||||
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
|
||||
log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
|
||||
}
|
||||
}
|
||||
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
|
||||
|
||||
isEmpty, err := gitRepo.IsEmpty()
|
||||
|
||||
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
|
||||
}
|
||||
|
||||
notify_service.ChangeDefaultBranch(ctx, repo)
|
||||
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
|
||||
}
|
||||
}
|
||||
|
||||
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
|
||||
}
|
||||
|
||||
return repo, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/gitrepo"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
)
|
||||
|
||||
const heatmapIndexCommitsPageSize = 100
|
||||
|
||||
// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch.
|
||||
func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error {
|
||||
if repo == nil || repo.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
if repo.DefaultBranch == "" || repo.IsEmpty {
|
||||
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
|
||||
}
|
||||
|
||||
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer.Close()
|
||||
|
||||
head, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
|
||||
if git.IsErrNotExist(err) {
|
||||
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0)
|
||||
now := timeutil.TimeStampNow()
|
||||
for page := 1; ; page++ {
|
||||
commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(commits) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, commit := range commits {
|
||||
commitSHA := commit.ID.String()
|
||||
if commit.Author == nil || commit.Author.Email == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
authorUnix := timeutil.TimeStamp(commit.Author.When.Unix())
|
||||
if authorUnix > now {
|
||||
continue
|
||||
}
|
||||
|
||||
userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{
|
||||
UserID: userID,
|
||||
RepoID: repo.ID,
|
||||
CommitSHA: commitSHA,
|
||||
AuthorEmail: strings.ToLower(commit.Author.Email),
|
||||
AuthorUnix: authorUnix,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{
|
||||
RepoID: repo.ID,
|
||||
CommitSHA: commitSHA,
|
||||
UserID: userID,
|
||||
})
|
||||
}
|
||||
|
||||
if len(commits) < heatmapIndexCommitsPageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities)
|
||||
}
|
||||
|
||||
func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) {
|
||||
address, err := user_model.GetEmailAddressByEmail(ctx, email)
|
||||
if user_model.IsErrEmailAddressNotExist(err) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if !address.IsActivated {
|
||||
return 0, false, nil
|
||||
}
|
||||
return address.UID, true, nil
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||
repo_module "code.gitea.io/gitea/modules/repository"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) {
|
||||
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{
|
||||
{Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||
{Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||
{Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"},
|
||||
})
|
||||
|
||||
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||
|
||||
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 1)
|
||||
assert.Equal(t, int64(2), contributions[0].UserID)
|
||||
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
|
||||
assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail)
|
||||
assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix)
|
||||
|
||||
emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com")
|
||||
require.NoError(t, err)
|
||||
emailAddress.UID = 1
|
||||
require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress))
|
||||
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||
|
||||
contributions = loadHeatmapContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 1)
|
||||
assert.Equal(t, int64(1), contributions[0].UserID)
|
||||
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
|
||||
}
|
||||
|
||||
func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
|
||||
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{
|
||||
{Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||
{Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||
{Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
|
||||
})
|
||||
|
||||
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||
|
||||
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 2)
|
||||
assert.Equal(t, int64(2), contributions[0].UserID)
|
||||
assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA)
|
||||
assert.Equal(t, int64(1), contributions[1].UserID)
|
||||
assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA)
|
||||
for _, contribution := range contributions {
|
||||
assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA)
|
||||
}
|
||||
|
||||
deleteMainRef(t, repo)
|
||||
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
|
||||
}
|
||||
|
||||
func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
|
||||
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
|
||||
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||
})
|
||||
|
||||
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||
assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
|
||||
|
||||
newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
|
||||
{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||
})
|
||||
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
|
||||
{
|
||||
RefFullName: git.RefNameFromBranch("main"),
|
||||
OldCommitID: commits["initial"],
|
||||
NewCommitID: newCommits["pushed"],
|
||||
PusherID: 1,
|
||||
RepoUserName: repo.OwnerName,
|
||||
RepoName: repo.Name,
|
||||
},
|
||||
}))
|
||||
|
||||
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||
require.Len(t, contributions, 2)
|
||||
assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
|
||||
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
|
||||
{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
|
||||
})
|
||||
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
|
||||
{
|
||||
RefFullName: git.RefNameFromBranch("feature"),
|
||||
OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(),
|
||||
NewCommitID: featureCommits["feature-pushed"],
|
||||
PusherID: 1,
|
||||
RepoUserName: repo.OwnerName,
|
||||
RepoName: repo.Name,
|
||||
},
|
||||
}))
|
||||
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
|
||||
}
|
||||
|
||||
type heatmapIndexTestCommit struct {
|
||||
Branch string
|
||||
Mark string
|
||||
Parent string
|
||||
AuthorName string
|
||||
AuthorEmail string
|
||||
CommitterName string
|
||||
CommitterEmail string
|
||||
AuthorDate string
|
||||
}
|
||||
|
||||
func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) {
|
||||
t.Helper()
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
|
||||
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||
repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
|
||||
require.NoError(t, err)
|
||||
|
||||
marks := runFastImport(t, repo, commits)
|
||||
repo.IsEmpty = false
|
||||
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
|
||||
return repo, marks
|
||||
}
|
||||
|
||||
func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
var stream strings.Builder
|
||||
branchTips := make(map[string]string)
|
||||
for i, commit := range commits {
|
||||
mark := fmt.Sprintf(":%d", i+1)
|
||||
branchRef := "refs/heads/" + commit.Branch
|
||||
stream.WriteString("commit " + branchRef + "\n")
|
||||
stream.WriteString("mark " + mark + "\n")
|
||||
stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
|
||||
stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
|
||||
message := "heatmap " + commit.Mark
|
||||
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
|
||||
if parentMark := branchTips[commit.Branch]; parentMark != "" {
|
||||
stream.WriteString("from " + parentMark + "\n")
|
||||
} else if commit.Parent != "" {
|
||||
stream.WriteString("from " + commit.Parent + "\n")
|
||||
}
|
||||
content := commit.Mark + "\n"
|
||||
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
|
||||
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
|
||||
branchTips[commit.Branch] = mark
|
||||
}
|
||||
|
||||
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
|
||||
WithDir(repo.RepoPath()).
|
||||
WithStdinCopy(strings.NewReader(stream.String())).
|
||||
Run(t.Context()))
|
||||
|
||||
commitSHAs := make(map[string]string, len(commits))
|
||||
for _, commit := range commits {
|
||||
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
|
||||
AddOptionFormat("--grep=%s", "heatmap "+commit.Mark).
|
||||
AddDynamicArguments("refs/heads/" + commit.Branch).
|
||||
WithDir(repo.RepoPath()).
|
||||
RunStdString(t.Context())
|
||||
require.NoError(t, err)
|
||||
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
|
||||
}
|
||||
return commitSHAs
|
||||
}
|
||||
|
||||
func signatureLine(kind, name, email, date string) string {
|
||||
parsed, err := time.Parse(time.RFC3339, date)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
|
||||
}
|
||||
|
||||
func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
|
||||
t.Helper()
|
||||
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
|
||||
require.NoError(t, err)
|
||||
return contributions
|
||||
}
|
||||
|
||||
func deleteMainRef(t *testing.T, repo *repo_model.Repository) {
|
||||
t.Helper()
|
||||
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
|
||||
}
|
||||
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
||||
}
|
||||
}
|
||||
|
||||
return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
|
||||
repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
|
||||
if opts.Mirror {
|
||||
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
|
||||
if err != nil {
|
||||
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
||||
}
|
||||
return repo, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
|
||||
}
|
||||
return repo, nil
|
||||
}
|
||||
|
||||
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
|
||||
|
||||
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
||||
}
|
||||
}
|
||||
|
||||
indexHeatmap := false
|
||||
if !opts.IsDelRef() {
|
||||
branch := opts.RefFullName.BranchName()
|
||||
|
||||
@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
||||
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
|
||||
log.Error("DelRepoDivergenceFromCache: %v", err)
|
||||
}
|
||||
indexHeatmap = true
|
||||
} else {
|
||||
if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
|
||||
log.Error("DelDivergenceFromCache: %v", err)
|
||||
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
||||
pushDeleteBranch(ctx, repo, pusher, opts)
|
||||
}
|
||||
|
||||
if indexHeatmap {
|
||||
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
||||
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
|
||||
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
auth_model "code.gitea.io/gitea/models/auth"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
password_module "code.gitea.io/gitea/modules/auth/password"
|
||||
"code.gitea.io/gitea/modules/optional"
|
||||
@@ -47,6 +48,7 @@ type UpdateOptions struct {
|
||||
IsRestricted optional.Option[bool]
|
||||
Visibility optional.Option[structs.VisibleType]
|
||||
KeepActivityPrivate optional.Option[bool]
|
||||
IncludePrivateContributions optional.Option[bool]
|
||||
Language optional.Option[string]
|
||||
Theme optional.Option[string]
|
||||
DiffViewStyle optional.Option[string]
|
||||
@@ -182,7 +184,23 @@ func UpdateUser(ctx context.Context, u *user_model.User, opts *UpdateOptions) er
|
||||
cols = append(cols, "last_login_unix")
|
||||
}
|
||||
|
||||
return user_model.UpdateUserCols(ctx, u, cols...)
|
||||
if len(cols) > 0 || opts.IncludePrivateContributions.Has() {
|
||||
return db.WithTx(ctx, func(ctx context.Context) error {
|
||||
if len(cols) > 0 {
|
||||
if err := user_model.UpdateUserCols(ctx, u, cols...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if opts.IncludePrivateContributions.Has() {
|
||||
return user_model.SetIncludePrivateContributions(ctx, u.ID, opts.IncludePrivateContributions.Value())
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type UpdateAuthOptions struct {
|
||||
|
||||
@@ -44,6 +44,7 @@ func TestUpdateUser(t *testing.T) {
|
||||
IsAdmin: UpdateOptionFieldFromValue(true),
|
||||
Visibility: optional.Some(structs.VisibleTypePrivate),
|
||||
KeepActivityPrivate: optional.Some(true),
|
||||
IncludePrivateContributions: optional.Some(true),
|
||||
Language: optional.Some("lang"),
|
||||
Theme: optional.Some("theme"),
|
||||
DiffViewStyle: optional.Some("split"),
|
||||
@@ -66,6 +67,9 @@ func TestUpdateUser(t *testing.T) {
|
||||
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
||||
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
||||
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
||||
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
|
||||
assert.Equal(t, opts.Language.Value(), user.Language)
|
||||
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
||||
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
||||
@@ -86,6 +90,9 @@ func TestUpdateUser(t *testing.T) {
|
||||
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
||||
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
||||
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
||||
includePrivateContributions, err = user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
|
||||
assert.Equal(t, opts.Language.Value(), user.Language)
|
||||
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
||||
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
||||
|
||||
@@ -29655,6 +29655,10 @@
|
||||
"type": "boolean",
|
||||
"x-go-name": "HideEmail"
|
||||
},
|
||||
"include_private_contributions": {
|
||||
"type": "boolean",
|
||||
"x-go-name": "IncludePrivateContributions"
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"x-go-name": "Language"
|
||||
@@ -29699,6 +29703,10 @@
|
||||
"type": "boolean",
|
||||
"x-go-name": "HideEmail"
|
||||
},
|
||||
"include_private_contributions": {
|
||||
"type": "boolean",
|
||||
"x-go-name": "IncludePrivateContributions"
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"x-go-name": "Language"
|
||||
|
||||
@@ -88,6 +88,13 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="field">
|
||||
<div class="ui checkbox" id="include-private-contributions">
|
||||
<label data-tooltip-content="{{ctx.Locale.Tr "settings.include_private_contributions_popup"}}"><strong>{{ctx.Locale.Tr "settings.include_private_contributions"}}</strong></label>
|
||||
<input name="include_private_contributions" type="checkbox" {{if .IncludePrivateContributions}}checked{{end}}>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="divider"></div>
|
||||
|
||||
<div class="field">
|
||||
|
||||
@@ -4,17 +4,19 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
auth_model "code.gitea.io/gitea/models/auth"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
"code.gitea.io/gitea/tests"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUserHeatmap(t *testing.T) {
|
||||
@@ -23,17 +25,23 @@ func TestUserHeatmap(t *testing.T) {
|
||||
normalUsername := "user2"
|
||||
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
|
||||
|
||||
fakeNow := time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local)
|
||||
fakeNow := time.Date(2011, 10, 21, 0, 0, 0, 0, time.Local)
|
||||
timeutil.MockSet(fakeNow)
|
||||
defer timeutil.MockUnset()
|
||||
|
||||
req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)).
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-author-date", 1319068800)
|
||||
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-same-bucket", 1319068850)
|
||||
testHeatmapSeedContribution(t, 2, 2, "api-user2-private-hidden", 1319070600)
|
||||
|
||||
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", normalUsername).
|
||||
AddTokenAuth(token)
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
var heatmap []*activities_model.UserHeatmapData
|
||||
DecodeJSON(t, resp, &heatmap)
|
||||
var dummyheatmap []*activities_model.UserHeatmapData
|
||||
dummyheatmap = append(dummyheatmap, &activities_model.UserHeatmapData{Timestamp: 1603227600, Contributions: 1})
|
||||
|
||||
assert.Equal(t, dummyheatmap, heatmap)
|
||||
assert.Equal(t, []*activities_model.UserHeatmapData{
|
||||
{Timestamp: 1319068800, Contributions: 2},
|
||||
}, heatmap)
|
||||
}
|
||||
|
||||
@@ -4,22 +4,40 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
"code.gitea.io/gitea/tests"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type testHeatmapWebResponse struct {
|
||||
HeatmapData [][2]int64 `json:"heatmapData"`
|
||||
TotalContributions int64 `json:"totalContributions"`
|
||||
}
|
||||
|
||||
func TestHeatmapEndpoints(t *testing.T) {
|
||||
defer tests.PrepareTestEnv(t)()
|
||||
|
||||
// Mock time so fixture actions fall within the heatmap's time window
|
||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
defer timeutil.MockUnset()
|
||||
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-one", 1603009800)
|
||||
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-two", 1603009850)
|
||||
testHeatmapSeedContribution(t, 2, 2, "web-user2-private-hidden", 1603010700)
|
||||
|
||||
session := loginUser(t, "user2")
|
||||
|
||||
@@ -28,11 +46,15 @@ func TestHeatmapEndpoints(t *testing.T) {
|
||||
req := NewRequest(t, "GET", "/user2/-/heatmap")
|
||||
resp := session.MakeRequest(t, req, http.StatusOK)
|
||||
|
||||
var result map[string]any
|
||||
DecodeJSON(t, resp, &result)
|
||||
assert.Contains(t, result, "heatmapData")
|
||||
assert.Contains(t, result, "totalContributions")
|
||||
assert.Positive(t, result["totalContributions"])
|
||||
webHeatmap := testHeatmapDecodeWebResponse(t, resp)
|
||||
|
||||
req = NewRequest(t, "GET", "/api/v1/users/user2/heatmap")
|
||||
resp = session.MakeRequest(t, req, http.StatusOK)
|
||||
var apiHeatmap []*activities_model.UserHeatmapData
|
||||
DecodeJSON(t, resp, &apiHeatmap)
|
||||
|
||||
assert.Equal(t, testHeatmapSumAPIContributions(apiHeatmap), webHeatmap.TotalContributions)
|
||||
assert.Equal(t, int64(2), webHeatmap.TotalContributions)
|
||||
})
|
||||
|
||||
t.Run("OrgDashboard", func(t *testing.T) {
|
||||
@@ -57,3 +79,71 @@ func TestHeatmapEndpoints(t *testing.T) {
|
||||
assert.Contains(t, result, "totalContributions")
|
||||
})
|
||||
}
|
||||
|
||||
func testHeatmapSeedContribution(t *testing.T, userID, repoID int64, label string, authorUnix timeutil.TimeStamp) string {
|
||||
t.Helper()
|
||||
|
||||
commitSHA := fmt.Sprintf("%040x", sha1.Sum([]byte(label)))
|
||||
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||
UserID: userID,
|
||||
RepoID: repoID,
|
||||
CommitSHA: commitSHA,
|
||||
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
|
||||
AuthorUnix: authorUnix,
|
||||
}))
|
||||
return commitSHA
|
||||
}
|
||||
|
||||
func testHeatmapDecodeWebResponse(t *testing.T, resp *httptest.ResponseRecorder) testHeatmapWebResponse {
|
||||
t.Helper()
|
||||
|
||||
var result testHeatmapWebResponse
|
||||
DecodeJSON(t, resp, &result)
|
||||
return result
|
||||
}
|
||||
|
||||
func testHeatmapSumAPIContributions(heatmap []*activities_model.UserHeatmapData) int64 {
|
||||
var total int64
|
||||
for _, item := range heatmap {
|
||||
total += item.Contributions
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func testHeatmapAssertAggregateOnlyAPIResponse(t *testing.T, body []byte) {
|
||||
t.Helper()
|
||||
|
||||
var decoded []map[string]any
|
||||
require.NoError(t, json.Unmarshal(body, &decoded))
|
||||
for _, entry := range decoded {
|
||||
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, testHeatmapMapKeys(entry))
|
||||
}
|
||||
}
|
||||
|
||||
func testHeatmapAssertAggregateOnlyWebResponse(t *testing.T, body []byte) {
|
||||
t.Helper()
|
||||
|
||||
var decoded map[string]any
|
||||
require.NoError(t, json.Unmarshal(body, &decoded))
|
||||
assert.ElementsMatch(t, []string{"heatmapData", "totalContributions"}, testHeatmapMapKeys(decoded))
|
||||
}
|
||||
|
||||
func testHeatmapAssertNoPrivateMetadata(t *testing.T, body []byte, forbidden ...string) {
|
||||
t.Helper()
|
||||
|
||||
response := string(body)
|
||||
for _, value := range forbidden {
|
||||
assert.NotContains(t, response, value)
|
||||
}
|
||||
for _, value := range []string{"repo_id", "repository_id", "repoID", "commit_sha", "commit_message", "branch", "url", "action_id"} {
|
||||
assert.NotContains(t, response, value)
|
||||
}
|
||||
}
|
||||
|
||||
func testHeatmapMapKeys[K comparable, V any](m map[K]V) []K {
|
||||
keys := make([]K, 0, len(m))
|
||||
for key := range m {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user