forked from hinterland/hearth
Compare commits
67 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 557b6e9ef0 | |||
| e49f497045 | |||
| feb1a48db1 | |||
| 30732080b7 | |||
| 80cf1588bb | |||
| ef7d1b29f4 | |||
| 7e8c6884db | |||
| 1dd41e608b | |||
| e41c3e5a05 | |||
| f473280bf5 | |||
| b08fdd6e6b | |||
| f73bfc63be | |||
| f6e7c1eca9 | |||
| 6996d178ef | |||
| 7f7229b199 | |||
| 2d5bd26c36 | |||
| fba150b55b | |||
| 2e7bf58acf | |||
| b12c35f957 | |||
| bcf1b84dc4 | |||
| 7c25e3b46d | |||
| a20381e343 | |||
| bd92610a98 | |||
| e3ee881db6 | |||
| b9eabca464 | |||
| fcc72192f5 | |||
| 5a0696ce64 | |||
| f4a59ff117 | |||
| 129c82c863 | |||
| 968c654320 | |||
| 98bb6c568f | |||
| 72168aa8fa | |||
| 28dde5b9b1 | |||
| c2e0ba200c | |||
| 2eb23ea7ea | |||
| 9906f71c5d | |||
| 593c0d9abc | |||
| 53dfd60b4c | |||
| 3299daf061 | |||
| 2856ca1d98 | |||
| e04b7e11da | |||
| 59dc5ecd1e | |||
| ad6c5ab803 | |||
| 592d1f04c5 | |||
| d76c0b0273 | |||
| 0914391a2b | |||
| d88c1cbb4f | |||
| 35a5d59cbe | |||
| 341e3a0e1c | |||
| a30d1a93dd | |||
| c50d274ae1 | |||
| df19d16269 | |||
| 35af6720ef | |||
| a33432d5de | |||
| 7152eb03de | |||
| 8e5ba8de7f | |||
| 7c10fda451 | |||
| ca88f92b1a | |||
| 5b5f119a65 | |||
| 8caf575946 | |||
| d644d390a7 | |||
| b56dc50e50 | |||
| 63223329dd | |||
| c74992ea85 | |||
| c0c024dcfd | |||
| 0023e27110 | |||
| 882b4ec871 |
@@ -0,0 +1,29 @@
|
|||||||
|
name: runner nix smoke
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- .gitea/workflows/runner-nix-smoke.yaml
|
||||||
|
- flake.lock
|
||||||
|
- flake.nix
|
||||||
|
- infra/gitea-runners/**
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: nix label smoke
|
||||||
|
runs-on: nix
|
||||||
|
steps:
|
||||||
|
- name: Nix version and cache configuration
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
nix --version
|
||||||
|
nix config show substituters
|
||||||
|
nix config show trusted-public-keys
|
||||||
|
|
||||||
|
- name: Repository flake evaluation
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
nix flake check --no-build
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
name: runner ubuntu smoke
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- .gitea/workflows/runner-ubuntu-smoke.yaml
|
||||||
|
- infra/gitea-runners/**
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: ubuntu-latest label smoke
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Basic runner information
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
echo "hello from gitea runner"
|
||||||
|
uname -a
|
||||||
|
|
||||||
|
- name: Docker smoke when available
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
|
||||||
|
docker run --rm hello-world
|
||||||
|
else
|
||||||
|
echo "docker command not available; skipping Docker smoke"
|
||||||
|
fi
|
||||||
@@ -47,6 +47,15 @@ creation_rules:
|
|||||||
- *hectic-lab-server
|
- *hectic-lab-server
|
||||||
- *umbriel-bfs
|
- *umbriel-bfs
|
||||||
|
|
||||||
|
- path_regex: sus/gitea-runners.yaml$
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *nrv
|
||||||
|
- *yukkop
|
||||||
|
- *yukkop-alt
|
||||||
|
- *hectic-lab-server
|
||||||
|
- *umbriel-bfs
|
||||||
|
|
||||||
- path_regex: sus/matrix-cluster.yaml$
|
- path_regex: sus/matrix-cluster.yaml$
|
||||||
key_groups:
|
key_groups:
|
||||||
- age:
|
- age:
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
flake,
|
||||||
|
self,
|
||||||
|
inputs,
|
||||||
|
system ? "aarch64-darwin",
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
name = builtins.baseNameOf ./.;
|
||||||
|
in inputs.nix-darwin.lib.darwinSystem {
|
||||||
|
inherit system;
|
||||||
|
specialArgs = { inherit flake self inputs; };
|
||||||
|
modules = [
|
||||||
|
inputs.home-manager.darwinModules.home-manager
|
||||||
|
{
|
||||||
|
networking.hostName = name;
|
||||||
|
nixpkgs.hostPlatform = system;
|
||||||
|
nixpkgs.overlays = [ self.overlays.default ];
|
||||||
|
}
|
||||||
|
./${name}.nix
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
{
|
||||||
|
flake,
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
name = "yukkop";
|
||||||
|
in {
|
||||||
|
system.primaryUser = name;
|
||||||
|
nix.settings.experimental-features = "nix-command flakes";
|
||||||
|
|
||||||
|
programs.zsh.enable = true;
|
||||||
|
|
||||||
|
services.openssh.enable = true;
|
||||||
|
|
||||||
|
users.users.${name} = {
|
||||||
|
home = "/Users/${name}";
|
||||||
|
openssh.authorizedKeys.keys = [
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = with pkgs; [
|
||||||
|
aerospace
|
||||||
|
git
|
||||||
|
moreutils
|
||||||
|
neovim
|
||||||
|
tmux
|
||||||
|
];
|
||||||
|
|
||||||
|
launchd.user.agents.aerospace = {
|
||||||
|
serviceConfig = {
|
||||||
|
ProgramArguments = [
|
||||||
|
"${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
|
||||||
|
];
|
||||||
|
RunAtLoad = true;
|
||||||
|
KeepAlive = true;
|
||||||
|
StandardOutPath = "/tmp/aerospace.out.log";
|
||||||
|
StandardErrorPath = "/tmp/aerospace.err.log";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
home-manager.useGlobalPkgs = true;
|
||||||
|
home-manager.useUserPackages = true;
|
||||||
|
home-manager.backupFileExtension = "backup";
|
||||||
|
home-manager.sharedModules = [
|
||||||
|
(flake + "/home/module/program/tmux.nix")
|
||||||
|
];
|
||||||
|
home-manager.users.${name} = {
|
||||||
|
home.stateVersion = "25.11";
|
||||||
|
|
||||||
|
home.packages = with pkgs; [
|
||||||
|
iproute2mac
|
||||||
|
jujutsu
|
||||||
|
ripgrep
|
||||||
|
];
|
||||||
|
|
||||||
|
programs.git = {
|
||||||
|
enable = true;
|
||||||
|
lfs.enable = true;
|
||||||
|
settings = {
|
||||||
|
user.name = name;
|
||||||
|
user.email = "hectic.yukkop@gmail.com";
|
||||||
|
push.autoSetupRemote = true;
|
||||||
|
init.defaultBranch = "master";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
programs.zsh = {
|
||||||
|
enable = true;
|
||||||
|
enableCompletion = true;
|
||||||
|
autosuggestion.enable = true;
|
||||||
|
syntaxHighlighting.enable = true;
|
||||||
|
|
||||||
|
history = {
|
||||||
|
size = 10000;
|
||||||
|
path = "$HOME/.zsh/.zsh_history";
|
||||||
|
};
|
||||||
|
|
||||||
|
shellAliases = {
|
||||||
|
drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
|
||||||
|
nv = "nvim";
|
||||||
|
tmux = "tmux a";
|
||||||
|
};
|
||||||
|
|
||||||
|
initContent = ''
|
||||||
|
export PATH=/Users/yukkop/.opencode/bin:$PATH
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
xdg.configFile."aerospace/aerospace.toml".text = ''
|
||||||
|
start-at-login = false
|
||||||
|
|
||||||
|
enable-normalization-flatten-containers = true
|
||||||
|
enable-normalization-opposite-orientation-for-nested-containers = true
|
||||||
|
|
||||||
|
default-root-container-layout = 'tiles'
|
||||||
|
default-root-container-orientation = 'auto'
|
||||||
|
accordion-padding = 30
|
||||||
|
|
||||||
|
on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
|
||||||
|
automatically-unhide-macos-hidden-apps = false
|
||||||
|
|
||||||
|
[exec]
|
||||||
|
inherit-env-vars = true
|
||||||
|
|
||||||
|
[exec.env-vars]
|
||||||
|
PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
|
||||||
|
|
||||||
|
[gaps]
|
||||||
|
inner.horizontal = 8
|
||||||
|
inner.vertical = 8
|
||||||
|
outer.left = 8
|
||||||
|
outer.bottom = 8
|
||||||
|
outer.top = 8
|
||||||
|
outer.right = 8
|
||||||
|
|
||||||
|
[mode.main.binding]
|
||||||
|
alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
|
||||||
|
|
||||||
|
alt-slash = 'layout tiles horizontal vertical'
|
||||||
|
alt-comma = 'layout accordion horizontal vertical'
|
||||||
|
alt-f = 'fullscreen'
|
||||||
|
|
||||||
|
alt-h = 'focus left'
|
||||||
|
alt-j = 'focus down'
|
||||||
|
alt-k = 'focus up'
|
||||||
|
alt-l = 'focus right'
|
||||||
|
|
||||||
|
alt-shift-h = 'move left'
|
||||||
|
alt-shift-j = 'move down'
|
||||||
|
alt-shift-k = 'move up'
|
||||||
|
alt-shift-l = 'move right'
|
||||||
|
|
||||||
|
alt-minus = 'resize smart -50'
|
||||||
|
alt-equal = 'resize smart +50'
|
||||||
|
|
||||||
|
alt-1 = 'workspace 1'
|
||||||
|
alt-2 = 'workspace 2'
|
||||||
|
alt-3 = 'workspace 3'
|
||||||
|
alt-4 = 'workspace 4'
|
||||||
|
alt-5 = 'workspace 5'
|
||||||
|
alt-6 = 'workspace 6'
|
||||||
|
alt-7 = 'workspace 7'
|
||||||
|
alt-8 = 'workspace 8'
|
||||||
|
alt-9 = 'workspace 9'
|
||||||
|
|
||||||
|
alt-shift-1 = 'move-node-to-workspace 1'
|
||||||
|
alt-shift-2 = 'move-node-to-workspace 2'
|
||||||
|
alt-shift-3 = 'move-node-to-workspace 3'
|
||||||
|
alt-shift-4 = 'move-node-to-workspace 4'
|
||||||
|
alt-shift-5 = 'move-node-to-workspace 5'
|
||||||
|
alt-shift-6 = 'move-node-to-workspace 6'
|
||||||
|
alt-shift-7 = 'move-node-to-workspace 7'
|
||||||
|
alt-shift-8 = 'move-node-to-workspace 8'
|
||||||
|
alt-shift-9 = 'move-node-to-workspace 9'
|
||||||
|
|
||||||
|
alt-tab = 'workspace-back-and-forth'
|
||||||
|
alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
|
||||||
|
|
||||||
|
alt-shift-semicolon = 'mode service'
|
||||||
|
|
||||||
|
[mode.service.binding]
|
||||||
|
esc = ['reload-config', 'mode main']
|
||||||
|
r = ['flatten-workspace-tree', 'mode main']
|
||||||
|
f = ['layout floating tiling', 'mode main']
|
||||||
|
b = ['balance-sizes', 'mode main']
|
||||||
|
backspace = ['close-all-windows-but-current', 'mode main']
|
||||||
|
|
||||||
|
alt-shift-h = ['join-with left', 'mode main']
|
||||||
|
alt-shift-j = ['join-with down', 'mode main']
|
||||||
|
alt-shift-k = ['join-with up', 'mode main']
|
||||||
|
alt-shift-l = ['join-with right', 'mode main']
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
system.stateVersion = 6;
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
haskell = import ./haskell.nix { inherit self system pkgs; };
|
haskell = import ./haskell.nix { inherit self system pkgs; };
|
||||||
neuro = import ./neuro.nix { inherit self system pkgs; };
|
neuro = import ./neuro.nix { inherit self system pkgs; };
|
||||||
xmpp = import ./xmpp.nix { inherit self system pkgs; };
|
xmpp = import ./xmpp.nix { inherit self system pkgs; };
|
||||||
|
gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
|
||||||
default = pkgs.mkShell {
|
default = pkgs.mkShell {
|
||||||
buildInputs =
|
buildInputs =
|
||||||
(with self.packages.${system}; [
|
(with self.packages.${system}; [
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
{ pkgs, ... }: let
|
||||||
|
opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
|
||||||
|
|
||||||
|
tofu = pkgs.writeShellScriptBin "tofu" ''
|
||||||
|
exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
|
||||||
|
'';
|
||||||
|
|
||||||
|
giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
|
||||||
|
cat <<'EOF'
|
||||||
|
Gitea runners setup checklist
|
||||||
|
|
||||||
|
Tools available in this shell:
|
||||||
|
tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
|
||||||
|
docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
|
||||||
|
|
||||||
|
Environment expected before real deploy/apply:
|
||||||
|
TF_VAR_hcloud_token
|
||||||
|
TF_VAR_ssh_public_key
|
||||||
|
TF_VAR_ssh_private_key
|
||||||
|
S3 backend credentials and endpoint access
|
||||||
|
a matching SOPS age identity for sus/gitea-runners.yaml
|
||||||
|
kubectl access to the target cluster
|
||||||
|
a concrete registry digest for the pushed Nix-capable runner image if enabling
|
||||||
|
the nix label
|
||||||
|
|
||||||
|
OpenTofu validation gate:
|
||||||
|
tofu version
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
|
||||||
|
Nix image build/publish/digest gate:
|
||||||
|
nix build .#gitea-runner-nix-image
|
||||||
|
publish the archive, then pin the registry-reported digest in the runner label
|
||||||
|
mapping
|
||||||
|
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
|
||||||
|
SOPS token Secret creation gate:
|
||||||
|
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
|
||||||
|
Cluster provision gate:
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||||
|
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||||
|
|
||||||
|
Kubernetes apply gate:
|
||||||
|
kubectl config current-context
|
||||||
|
kubectl get nodes -o wide
|
||||||
|
kubectl get sc
|
||||||
|
kubectl apply -k infra/gitea-runners/k8s
|
||||||
|
|
||||||
|
Verification commands:
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
|
||||||
|
Main blockers and gates:
|
||||||
|
do not run tofu apply without all external inputs
|
||||||
|
do not apply the k8s overlay until the gitea-runner-token Secret exists
|
||||||
|
do not enable the nix label until the image has been published with a concrete digest
|
||||||
|
do not print, load, or require secrets on shell entry
|
||||||
|
EOF
|
||||||
|
'';
|
||||||
|
in pkgs.mkShell {
|
||||||
|
name = "gitea-runners";
|
||||||
|
|
||||||
|
buildInputs = [
|
||||||
|
tofu
|
||||||
|
giteaRunnersSetup
|
||||||
|
pkgs.nix
|
||||||
|
pkgs.kubectl
|
||||||
|
pkgs.kustomize
|
||||||
|
pkgs.kubeconform
|
||||||
|
pkgs.sops
|
||||||
|
pkgs.age
|
||||||
|
pkgs.awscli2
|
||||||
|
pkgs.hcloud
|
||||||
|
pkgs.tea
|
||||||
|
pkgs.docker
|
||||||
|
pkgs.skopeo
|
||||||
|
pkgs.go-containerregistry
|
||||||
|
pkgs.jq
|
||||||
|
pkgs.yq-go
|
||||||
|
pkgs.curl
|
||||||
|
pkgs.git
|
||||||
|
pkgs.openssh
|
||||||
|
];
|
||||||
|
|
||||||
|
shellHook = ''
|
||||||
|
export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
|
||||||
|
export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
|
||||||
|
export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
|
||||||
|
export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
|
||||||
|
export GITEA_RUNNERS_NAMESPACE="gitea-runners"
|
||||||
|
|
||||||
|
alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
|
||||||
|
alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
|
||||||
|
alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Gitea runner setup DevShell ==="
|
||||||
|
echo ""
|
||||||
|
echo "Run gitea-runners-setup for the full setup checklist."
|
||||||
|
echo "Paths: "
|
||||||
|
echo " root=$GITEA_RUNNERS_ROOT"
|
||||||
|
echo " tofu=$GITEA_RUNNERS_TOFU_DIR"
|
||||||
|
echo " k8s=$GITEA_RUNNERS_K8S_DIR"
|
||||||
|
echo " image=$GITEA_RUNNERS_IMAGE_DIR"
|
||||||
|
echo ""
|
||||||
|
'';
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
# Documentation
|
||||||
|
|
||||||
|
- [Using the `hectic` Attic Cache](./attic-cache.md)
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
# Using the `hectic` Attic Cache
|
||||||
|
|
||||||
|
This document explains how to:
|
||||||
|
|
||||||
|
1. pull build artifacts from the cache
|
||||||
|
2. push new artifacts to the cache
|
||||||
|
3. configure this flake to use the cache
|
||||||
|
|
||||||
|
## Cache endpoints
|
||||||
|
|
||||||
|
- API endpoint: `https://cache.hectic-lab.com`
|
||||||
|
- Binary cache endpoint: `https://cache.hectic-lab.com/hectic`
|
||||||
|
|
||||||
|
The `hectic` cache is:
|
||||||
|
|
||||||
|
- public for reads
|
||||||
|
- private for pushes
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
Use the Attic client package:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client
|
||||||
|
```
|
||||||
|
|
||||||
|
Or run commands directly with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c <command>
|
||||||
|
```
|
||||||
|
|
||||||
|
## Read from the cache
|
||||||
|
|
||||||
|
### Get the cache public key
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic cache info hectic
|
||||||
|
```
|
||||||
|
|
||||||
|
Copy the `Public Key` value, which looks like:
|
||||||
|
|
||||||
|
```text
|
||||||
|
hectic:...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configure Nix to trust the cache
|
||||||
|
|
||||||
|
Per-user: `~/.config/nix/nix.conf`
|
||||||
|
|
||||||
|
```ini
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
|
||||||
|
```
|
||||||
|
|
||||||
|
System-wide: `/etc/nix/nix.conf`
|
||||||
|
|
||||||
|
```ini
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
|
||||||
|
```
|
||||||
|
|
||||||
|
After that, normal Nix commands can download from the cache automatically:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#migrator
|
||||||
|
nix develop
|
||||||
|
nix flake check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use the cache from this flake
|
||||||
|
|
||||||
|
You can also advertise the cache from `flake.nix`:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
nixConfig = {
|
||||||
|
extra-substituters = [
|
||||||
|
"https://cache.nixos.org"
|
||||||
|
"https://cache.hectic-lab.com/hectic"
|
||||||
|
];
|
||||||
|
extra-trusted-public-keys = [
|
||||||
|
"hectic:PASTE_PUBLIC_KEY_HERE"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Then users can run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build --accept-flake-config .#migrator
|
||||||
|
```
|
||||||
|
|
||||||
|
## Log in for pushing
|
||||||
|
|
||||||
|
Pushing requires an Attic token.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
|
||||||
|
```
|
||||||
|
|
||||||
|
Example with `pass`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "$(pass show atticd/hectic-lab/token)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Push build results
|
||||||
|
|
||||||
|
### Push a package
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#migrator
|
||||||
|
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||||
|
```
|
||||||
|
|
||||||
|
### Push a check
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#checks.x86_64-linux.arguments
|
||||||
|
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||||
|
```
|
||||||
|
|
||||||
|
### Push a NixOS system build
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build '.#nixosConfigurations."hectic-lab|x86_64-linux".config.system.build.toplevel'
|
||||||
|
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||||
|
```
|
||||||
|
|
||||||
|
## Recommended workflow
|
||||||
|
|
||||||
|
### Local development
|
||||||
|
|
||||||
|
Use the cache for reads only:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#migrator
|
||||||
|
nix develop
|
||||||
|
nix flake check
|
||||||
|
```
|
||||||
|
|
||||||
|
### CI / builder
|
||||||
|
|
||||||
|
1. Build
|
||||||
|
2. Push to Attic
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#migrator
|
||||||
|
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||||
|
```
|
||||||
|
|
||||||
|
## Useful commands
|
||||||
|
|
||||||
|
### Show cache info
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic cache info hectic
|
||||||
|
```
|
||||||
|
|
||||||
|
### Check login config
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic cache info local:hectic
|
||||||
|
```
|
||||||
|
|
||||||
|
### Re-login with a new token
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<NEW_TOKEN>"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Common issues
|
||||||
|
|
||||||
|
### `flake 'nixpkgs' does not provide attribute 'attic'`
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client
|
||||||
|
```
|
||||||
|
|
||||||
|
Not:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic
|
||||||
|
```
|
||||||
|
|
||||||
|
### `HTTP 413 Payload Too Large`
|
||||||
|
|
||||||
|
This means nginx rejected the upload body size. The server must allow large uploads on the Attic vhost.
|
||||||
|
|
||||||
|
### Push succeeds for some paths but fails for others
|
||||||
|
|
||||||
|
Usually means:
|
||||||
|
|
||||||
|
- nginx body size limit
|
||||||
|
- timeout/reverse proxy issue
|
||||||
|
- bad token permissions
|
||||||
|
|
||||||
|
### Cache pulls do not work
|
||||||
|
|
||||||
|
Check:
|
||||||
|
|
||||||
|
- `substituters`
|
||||||
|
- `trusted-public-keys`
|
||||||
|
- the exact public key from `attic cache info hectic`
|
||||||
|
|
||||||
|
## Notes about retention and storage
|
||||||
|
|
||||||
|
- The cache currently uses Hetzner Object Storage
|
||||||
|
- If no `retention-period` is configured, cached objects do not expire automatically
|
||||||
|
- This is good for long-lived reuse, but storage usage can grow over time
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
### Read access
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#migrator
|
||||||
|
```
|
||||||
|
|
||||||
|
after configuring:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
|
||||||
|
```
|
||||||
|
|
||||||
|
### Push access
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
|
||||||
|
nix build .#migrator
|
||||||
|
nix shell nixpkgs#attic-client -c attic push local:hectic ./result
|
||||||
|
```
|
||||||
Generated
+36
-31
@@ -326,7 +326,7 @@
|
|||||||
"hectic-landing": {
|
"hectic-landing": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
"nixpkgs-fixed"
|
"nixpkgs"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
@@ -346,7 +346,7 @@
|
|||||||
"home-manager": {
|
"home-manager": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
"nixpkgs-fixed"
|
"nixpkgs"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
@@ -671,15 +671,15 @@
|
|||||||
"mechabellum-replay-analysis": {
|
"mechabellum-replay-analysis": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
"nixpkgs-fixed"
|
"nixpkgs"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1779576166,
|
"lastModified": 1780905541,
|
||||||
"narHash": "sha256-5bSuXkQs7KdbaYwDTdwUFlqOccVjPI2y42TZVq8lsNg=",
|
"narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
|
||||||
"ref": "refs/heads/master",
|
"ref": "refs/heads/master",
|
||||||
"rev": "f00295225c0dade61fe18b32262970c2665fb5fe",
|
"rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
|
||||||
"revCount": 110,
|
"revCount": 123,
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||||
},
|
},
|
||||||
@@ -688,6 +688,27 @@
|
|||||||
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
"url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"nix-darwin": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1772129556,
|
||||||
|
"narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
|
||||||
|
"owner": "nix-darwin",
|
||||||
|
"repo": "nix-darwin",
|
||||||
|
"rev": "ebec37af18215214173c98cf6356d0aca24a2585",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-darwin",
|
||||||
|
"ref": "nix-darwin-25.11",
|
||||||
|
"repo": "nix-darwin",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nix-minecraft": {
|
"nix-minecraft": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"flake-compat": "flake-compat_3",
|
"flake-compat": "flake-compat_3",
|
||||||
@@ -872,29 +893,13 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nixpkgs-fixed": {
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1771419570,
|
|
||||||
"narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
|
|
||||||
"owner": "NixOS",
|
|
||||||
"repo": "nixpkgs",
|
|
||||||
"rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"owner": "NixOS",
|
|
||||||
"ref": "nixos-25.11",
|
|
||||||
"repo": "nixpkgs",
|
|
||||||
"type": "github"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nixpkgs_2": {
|
"nixpkgs_2": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1771419570,
|
"lastModified": 1779796641,
|
||||||
"narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
|
"narHash": "sha256-ZsIrKmhp4vbBXoXXmR/tBXA/UCsAQiJL9vsgZEduhVY=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
|
"rev": "25f538306313eae3927264466c70d7001dcea1df",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -908,7 +913,7 @@
|
|||||||
"inputs": {
|
"inputs": {
|
||||||
"flake-parts": "flake-parts_2",
|
"flake-parts": "flake-parts_2",
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
"nixpkgs-fixed"
|
"nixpkgs"
|
||||||
],
|
],
|
||||||
"nuschtosSearch": "nuschtosSearch",
|
"nuschtosSearch": "nuschtosSearch",
|
||||||
"systems": "systems_5"
|
"systems": "systems_5"
|
||||||
@@ -983,13 +988,13 @@
|
|||||||
"hyprland": "hyprland",
|
"hyprland": "hyprland",
|
||||||
"impermanence": "impermanence",
|
"impermanence": "impermanence",
|
||||||
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
|
"mechabellum-replay-analysis": "mechabellum-replay-analysis",
|
||||||
|
"nix-darwin": "nix-darwin",
|
||||||
"nix-minecraft": "nix-minecraft",
|
"nix-minecraft": "nix-minecraft",
|
||||||
"nixos-anywhere": "nixos-anywhere",
|
"nixos-anywhere": "nixos-anywhere",
|
||||||
"nixos-hardware": "nixos-hardware",
|
"nixos-hardware": "nixos-hardware",
|
||||||
"nixos-mailserver": "nixos-mailserver",
|
"nixos-mailserver": "nixos-mailserver",
|
||||||
"nixos-wsl": "nixos-wsl",
|
"nixos-wsl": "nixos-wsl",
|
||||||
"nixpkgs": "nixpkgs_2",
|
"nixpkgs": "nixpkgs_2",
|
||||||
"nixpkgs-fixed": "nixpkgs-fixed",
|
|
||||||
"nixvim": "nixvim",
|
"nixvim": "nixvim",
|
||||||
"rust-overlay": "rust-overlay",
|
"rust-overlay": "rust-overlay",
|
||||||
"sops-nix": "sops-nix"
|
"sops-nix": "sops-nix"
|
||||||
@@ -1002,11 +1007,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1738290352,
|
"lastModified": 1780629589,
|
||||||
"narHash": "sha256-YKOHUmc0Clm4tMV8grnxYL4IIwtjTayoq/3nqk0QM7k=",
|
"narHash": "sha256-oHysjxZdaEqkmyDpN8G1bl3V+r9uyRD1O66bH0bq0Cs=",
|
||||||
"owner": "oxalica",
|
"owner": "oxalica",
|
||||||
"repo": "rust-overlay",
|
"repo": "rust-overlay",
|
||||||
"rev": "b031b584125d33d23a0182f91ddbaf3ab4880236",
|
"rev": "7a5a1c0a5cb86a28224304309b68f050835fd1f6",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
|||||||
@@ -1,8 +1,18 @@
|
|||||||
{
|
{
|
||||||
description = "yukkop's nix utilities";
|
description = "yukkop's nix utilities";
|
||||||
|
|
||||||
|
nixConfig = {
|
||||||
|
extra-substituters = [
|
||||||
|
"https://cache.nixos.org"
|
||||||
|
"https://cache.hectic-lab.com/hectic"
|
||||||
|
];
|
||||||
|
extra-trusted-public-keys = [
|
||||||
|
"hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
inputs = {
|
inputs = {
|
||||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
|
||||||
nixpkgs-fixed.url = "github:NixOS/nixpkgs/nixos-25.11";
|
|
||||||
rust-overlay = {
|
rust-overlay = {
|
||||||
url = "github:oxalica/rust-overlay";
|
url = "github:oxalica/rust-overlay";
|
||||||
inputs = {
|
inputs = {
|
||||||
@@ -19,7 +29,7 @@
|
|||||||
};
|
};
|
||||||
nixvim = {
|
nixvim = {
|
||||||
url = "github:nix-community/nixvim/nixos-25.11";
|
url = "github:nix-community/nixvim/nixos-25.11";
|
||||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
disko = {
|
disko = {
|
||||||
url = "github:nix-community/disko";
|
url = "github:nix-community/disko";
|
||||||
@@ -30,7 +40,11 @@
|
|||||||
};
|
};
|
||||||
home-manager = {
|
home-manager = {
|
||||||
url = "github:nix-community/home-manager/release-25.11";
|
url = "github:nix-community/home-manager/release-25.11";
|
||||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
nix-darwin = {
|
||||||
|
url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
nixos-wsl = {
|
nixos-wsl = {
|
||||||
url = "github:nix-community/NixOS-WSL";
|
url = "github:nix-community/NixOS-WSL";
|
||||||
@@ -56,13 +70,13 @@
|
|||||||
# NOTE(yukkop): private repo - SSH access required.
|
# NOTE(yukkop): private repo - SSH access required.
|
||||||
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
||||||
url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
|
url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
|
||||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
mechabellum-replay-analysis = {
|
mechabellum-replay-analysis = {
|
||||||
# NOTE(yukkop): private repo - SSH access required.
|
# NOTE(yukkop): private repo - SSH access required.
|
||||||
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
# Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
|
||||||
url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git";
|
url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git";
|
||||||
inputs.nixpkgs.follows = "nixpkgs-fixed";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -109,8 +123,12 @@
|
|||||||
# FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this
|
# FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this
|
||||||
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
|
"neuro|x86_64-linux" = import ./nixos/system/neuro { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
|
"games|x86_64-linux" = import ./nixos/system/games { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
|
"wsl|x86_64-linux" = import ./nixos/system/wsl { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
|
"tenix|x86_64-linux" = import ./nixos/system/tenix { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
|
"hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
|
||||||
|
};
|
||||||
|
darwinConfigurations = {
|
||||||
|
"yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{ pkgs, ... }: {
|
||||||
|
programs.tmux = {
|
||||||
|
enable = true;
|
||||||
|
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
|
||||||
|
keyMode = "vi";
|
||||||
|
escapeTime = 500;
|
||||||
|
historyLimit = 50000;
|
||||||
|
newSession = true;
|
||||||
|
extraConfig = ''
|
||||||
|
# resurrect
|
||||||
|
set -g @resurrect-strategy-vim 'session'
|
||||||
|
set -g @resurrect-strategy-nvim 'session'
|
||||||
|
set -g @resurrect-capture-pane-contents 'on'
|
||||||
|
|
||||||
|
resurrect_dir="$HOME/.tmux/resurrect"
|
||||||
|
set -g @resurrect-dir $resurrect_dir
|
||||||
|
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
|
||||||
|
|
||||||
|
# continuum
|
||||||
|
set -g @continuum-restore 'on'
|
||||||
|
set -g @continuum-boot 'on'
|
||||||
|
set -g @continuum-save-interval '10'
|
||||||
|
|
||||||
|
bind-key -T copy-mode-vi v send-keys -X begin-selection
|
||||||
|
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
|
||||||
|
|
||||||
|
bind-key O select-pane -t :.-
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
# Gitea runner Nix image
|
||||||
|
|
||||||
|
The repo-owned Nix-capable job image is built by the flake package
|
||||||
|
`gitea-runner-nix-image`.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix build .#gitea-runner-nix-image
|
||||||
|
```
|
||||||
|
|
||||||
|
The package emits a Docker archive with the local build tag:
|
||||||
|
|
||||||
|
```text
|
||||||
|
gitea-runner-nix-image:2026-06-07
|
||||||
|
```
|
||||||
|
|
||||||
|
That tag is build metadata only. Do not use it as the final Gitea runner label
|
||||||
|
mapping because runner job images must be immutable.
|
||||||
|
|
||||||
|
## Publication target
|
||||||
|
|
||||||
|
Preferred registry:
|
||||||
|
|
||||||
|
```text
|
||||||
|
gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish the archive without adding secrets to the image layers, then use the
|
||||||
|
registry-reported digest as the only final `nix` label image reference:
|
||||||
|
|
||||||
|
```text
|
||||||
|
nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
```
|
||||||
|
|
||||||
|
The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
|
||||||
|
runner label mapping must use the `@sha256:` reference above. Keep
|
||||||
|
`ubuntu-latest` on the `gitea/runner` default image unless a later runner
|
||||||
|
configuration task explicitly changes it. Only the `nix` label should select
|
||||||
|
this custom image.
|
||||||
|
|
||||||
|
If the Gitea container registry is unavailable, select a private registry that
|
||||||
|
is reachable from the runner Kubernetes cluster and requires authentication that
|
||||||
|
can be provided through Kubernetes image-pull secrets. Record the selected
|
||||||
|
registry and replace the host in the same digest-pinned form:
|
||||||
|
|
||||||
|
```text
|
||||||
|
nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not fall back to `latest` or a tag-only mapping.
|
||||||
|
|
||||||
|
## Task 7 publication status
|
||||||
|
|
||||||
|
Local build evidence is recorded in
|
||||||
|
`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
|
||||||
|
load and tag the image, but pushing to the preferred registry failed with
|
||||||
|
`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
|
||||||
|
concrete final mapping. Kubernetes pull smoke is recorded in
|
||||||
|
`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
|
||||||
|
is not installed or not on `PATH`.
|
||||||
|
|
||||||
|
Once registry credentials are available, rerun the push, capture the
|
||||||
|
registry-reported digest, and replace `<registry-digest>` in the mapping above
|
||||||
|
before Task 6/9 consumes the label configuration.
|
||||||
|
|
||||||
|
## Image contents
|
||||||
|
|
||||||
|
The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
|
||||||
|
`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
|
||||||
|
top-level `flake.nix`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
experimental-features = nix-command flakes
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||||
|
sandbox = false
|
||||||
|
```
|
||||||
|
|
||||||
|
No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
|
||||||
|
credential belongs in this image. Runtime secrets stay with the Kubernetes
|
||||||
|
runner configuration and token-file mount contract.
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- persistentvolumeclaims
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- apiGroups:
|
||||||
|
- apps
|
||||||
|
resources:
|
||||||
|
- statefulsets
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
data:
|
||||||
|
cleanup-dry-run.sh: |
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
namespace="${RUNNER_NAMESPACE:-gitea-runners}"
|
||||||
|
mode="${CLEANUP_MODE:-dry-run}"
|
||||||
|
selector="app.kubernetes.io/name=gitea-runner"
|
||||||
|
|
||||||
|
if [ "$namespace" != "gitea-runners" ]; then
|
||||||
|
printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
|
||||||
|
exit 13
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$mode" != "dry-run" ]; then
|
||||||
|
printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
|
||||||
|
exit 13
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'gitea runner lifecycle cleanup dry-run\n'
|
||||||
|
printf 'namespace: %s\n' "$namespace"
|
||||||
|
printf 'mode: %s\n\n' "$mode"
|
||||||
|
|
||||||
|
printf 'StatefulSet:\n'
|
||||||
|
kubectl -n "$namespace" get statefulset gitea-runner -o wide
|
||||||
|
|
||||||
|
printf '\nActive runner pods:\n'
|
||||||
|
kubectl -n "$namespace" get pods -l "$selector" -o wide
|
||||||
|
|
||||||
|
printf '\nRunner /data PVCs:\n'
|
||||||
|
kubectl -n "$namespace" get pvc -l "$selector" -o wide
|
||||||
|
|
||||||
|
printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
|
||||||
|
found_candidate=0
|
||||||
|
for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
pod="${pvc#data-}"
|
||||||
|
if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
|
||||||
|
found_candidate=1
|
||||||
|
printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$found_candidate" -eq 0 ]; then
|
||||||
|
printf 'none\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '\nGitea registration reconciliation:\n'
|
||||||
|
printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
|
||||||
|
printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-cleanup-dry-run
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
schedule: "17 3 * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
successfulJobsHistoryLimit: 3
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 3600
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner-lifecycle
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
serviceAccountName: gitea-runner-lifecycle
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: cleanup-dry-run
|
||||||
|
image: bitnami/kubectl:1.30
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- /scripts/cleanup-dry-run.sh
|
||||||
|
env:
|
||||||
|
- name: RUNNER_NAMESPACE
|
||||||
|
value: gitea-runners
|
||||||
|
- name: CLEANUP_MODE
|
||||||
|
value: dry-run
|
||||||
|
volumeMounts:
|
||||||
|
- name: lifecycle-scripts
|
||||||
|
mountPath: /scripts
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: lifecycle-scripts
|
||||||
|
configMap:
|
||||||
|
name: gitea-runner-lifecycle
|
||||||
|
defaultMode: 0555
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
- service.yaml
|
||||||
|
- service-account.yaml
|
||||||
|
- runner-config.yaml
|
||||||
|
- statefulset.yaml
|
||||||
|
- cleanup-lifecycle.yaml
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-config
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
data:
|
||||||
|
config.yaml: |
|
||||||
|
log:
|
||||||
|
level: info
|
||||||
|
|
||||||
|
runner:
|
||||||
|
file: /data/.runner
|
||||||
|
capacity: 1
|
||||||
|
envs: {}
|
||||||
|
timeout: 3h
|
||||||
|
insecure: false
|
||||||
|
fetch_timeout: 5s
|
||||||
|
fetch_interval: 2s
|
||||||
|
labels:
|
||||||
|
- ubuntu-latest
|
||||||
|
# The nix label is intentionally disabled until the runner image has a
|
||||||
|
# concrete registry-reported digest; see ../runbook.md before deploy.
|
||||||
|
|
||||||
|
cache:
|
||||||
|
enabled: true
|
||||||
|
dir: /data/cache
|
||||||
|
|
||||||
|
container:
|
||||||
|
network: bridge
|
||||||
|
privileged: false
|
||||||
|
force_pull: true
|
||||||
|
valid_volumes: []
|
||||||
|
docker_host: unix:///runner-docker/docker.sock
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
rules: []
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: gitea-runner
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
ports:
|
||||||
|
- name: cache
|
||||||
|
port: 8088
|
||||||
|
targetPort: cache
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner
|
||||||
|
namespace: gitea-runners
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
serviceName: gitea-runner
|
||||||
|
replicas: 5
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
annotations:
|
||||||
|
hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
|
||||||
|
spec:
|
||||||
|
serviceAccountName: gitea-runner
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
securityContext:
|
||||||
|
fsGroup: 1000
|
||||||
|
containers:
|
||||||
|
- name: runner
|
||||||
|
image: gitea/act_runner:0.2.11
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: GITEA_INSTANCE_URL
|
||||||
|
value: https://gitea.hectic-lab.com
|
||||||
|
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
||||||
|
value: /runner-secrets/token
|
||||||
|
- name: CONFIG_FILE
|
||||||
|
value: /runner-config/config.yaml
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: unix:///runner-docker/docker.sock
|
||||||
|
ports:
|
||||||
|
- name: cache
|
||||||
|
containerPort: 8088
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- test -s /data/.runner && test -S /runner-docker/docker.sock
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 6
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- test -S /runner-docker/docker.sock
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 6
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
- name: config
|
||||||
|
mountPath: /runner-config
|
||||||
|
readOnly: true
|
||||||
|
- name: runner-token
|
||||||
|
mountPath: /runner-secrets
|
||||||
|
readOnly: true
|
||||||
|
- name: docker-socket
|
||||||
|
mountPath: /runner-docker
|
||||||
|
- name: docker
|
||||||
|
image: docker:27-dind
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- --host=unix:///runner-docker/docker.sock
|
||||||
|
- --storage-driver=overlay2
|
||||||
|
- --tls=false
|
||||||
|
env:
|
||||||
|
- name: DOCKER_TLS_CERTDIR
|
||||||
|
value: ""
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: unix:///runner-docker/docker.sock
|
||||||
|
securityContext:
|
||||||
|
# Privileged rootful DinD is intentionally scoped to this trusted
|
||||||
|
# internal runner pool; never expose it to untrusted fork/PR jobs.
|
||||||
|
privileged: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 4Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- docker
|
||||||
|
- info
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 6
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- docker
|
||||||
|
- info
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 6
|
||||||
|
volumeMounts:
|
||||||
|
- name: docker-socket
|
||||||
|
mountPath: /runner-docker
|
||||||
|
- name: docker-graph
|
||||||
|
mountPath: /var/lib/docker
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: gitea-runner-config
|
||||||
|
- name: runner-token
|
||||||
|
secret:
|
||||||
|
secretName: gitea-runner-token
|
||||||
|
items:
|
||||||
|
- key: token
|
||||||
|
path: token
|
||||||
|
defaultMode: 0400
|
||||||
|
- name: docker-socket
|
||||||
|
emptyDir: {}
|
||||||
|
- name: docker-graph
|
||||||
|
emptyDir: {}
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: gitea-runner
|
||||||
|
app.kubernetes.io/part-of: gitea-actions
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: hcloud-volumes
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# OpenTofu working directory and downloaded modules/providers.
|
||||||
|
.terraform/
|
||||||
|
.terraform.lock.hcl
|
||||||
|
|
||||||
|
# State must live in the S3 backend for production. Local state is allowed only
|
||||||
|
# for throwaway syntax checks with `tofu init -backend=false` and must not be
|
||||||
|
# committed.
|
||||||
|
terraform.tfstate
|
||||||
|
terraform.tfstate.*
|
||||||
|
*.tfstate
|
||||||
|
*.tfstate.*
|
||||||
|
crash.log
|
||||||
|
crash.*.log
|
||||||
|
|
||||||
|
# Plans can contain secrets or derived infrastructure data.
|
||||||
|
*.tfplan
|
||||||
|
*.plan
|
||||||
|
kubeconfig
|
||||||
|
kubeconfig.yaml
|
||||||
|
*_kubeconfig.yaml
|
||||||
|
|
||||||
|
# Variable files commonly carry credentials. Keep production inputs in SOPS or
|
||||||
|
# external environment/configuration, not in checked-in files.
|
||||||
|
*.tfvars
|
||||||
|
*.tfvars.json
|
||||||
|
override.tf
|
||||||
|
override.tf.json
|
||||||
|
*_override.tf
|
||||||
|
*_override.tf.json
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# Gitea runner OpenTofu backend contract
|
||||||
|
|
||||||
|
This directory defines the safe backend, provider contract, and kube-hetzner
|
||||||
|
cluster stack for the Gitea runner Kubernetes cluster.
|
||||||
|
|
||||||
|
## Required backend
|
||||||
|
|
||||||
|
Production state must use the OpenTofu S3 backend in `backend.tf`:
|
||||||
|
|
||||||
|
- bucket: `gitea-runner-hectic-lab`
|
||||||
|
- key: `gitea-runners/kube-hetzner/terraform.tfstate`
|
||||||
|
- region: `fsn1`, aligned with the target Hetzner location
|
||||||
|
- encryption: `encrypt = true`
|
||||||
|
- locking: `use_lockfile = true` where the selected S3-compatible endpoint
|
||||||
|
supports it
|
||||||
|
|
||||||
|
Before any production `tofu init`, verify the S3-compatible endpoint, credential
|
||||||
|
source, bucket versioning, encryption behavior, and lockfile support for the
|
||||||
|
chosen object-storage provider. Keep backend authentication externalized through
|
||||||
|
environment variables, AWS-compatible shared config, or the production secret
|
||||||
|
injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
|
||||||
|
tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
|
||||||
|
OpenTofu files.
|
||||||
|
|
||||||
|
## Local state safety
|
||||||
|
|
||||||
|
Production local state is forbidden. Only syntax-only validation/prototyping may
|
||||||
|
use local state, and it must use backend-disabled initialization:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init -backend=false
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
```
|
||||||
|
|
||||||
|
Fail the run if production local state appears:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test ! -e infra/gitea-runners/opentofu/terraform.tfstate
|
||||||
|
test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
|
||||||
|
grep -R 'backend "s3"' infra/gitea-runners/opentofu
|
||||||
|
```
|
||||||
|
|
||||||
|
The `.gitignore` in this directory blocks local state, plans, downloaded
|
||||||
|
providers/modules, and variable files from being committed. Treat any local
|
||||||
|
state file as disposable validation residue, never as production state.
|
||||||
|
|
||||||
|
## Provider and module pins
|
||||||
|
|
||||||
|
`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
|
||||||
|
`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
|
||||||
|
observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
|
||||||
|
module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
|
||||||
|
as locals so Task 5 can wire the module without re-opening the version contract.
|
||||||
|
|
||||||
|
`providers.tf` leaves the `hcloud` provider empty so authentication comes from
|
||||||
|
the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
|
||||||
|
Do not set token values in `.tf` or `.tfvars` files.
|
||||||
|
|
||||||
|
## Cluster shape
|
||||||
|
|
||||||
|
The default cluster is deliberately fixed-size:
|
||||||
|
|
||||||
|
- cluster name: `gitea-runners`
|
||||||
|
- Hetzner location: `fsn1`
|
||||||
|
- private network region: `eu-central`
|
||||||
|
- control plane: one `cpx21` node in pool `control-plane`
|
||||||
|
- workers: three `cpx31` nodes in pool `runner-workers`
|
||||||
|
- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
|
||||||
|
- Longhorn: disabled
|
||||||
|
- autoscaling/KEDA: not enabled in this stack
|
||||||
|
|
||||||
|
The three default workers are sized for the initial five trusted privileged DinD
|
||||||
|
jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
|
||||||
|
`worker_count` to `5` or increase `worker_server_type`, then run a fresh
|
||||||
|
`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
|
||||||
|
|
||||||
|
Required inputs must come from environment or secret injection, for example
|
||||||
|
`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
|
||||||
|
Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
|
||||||
|
kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
|
||||||
|
enabled; this path is ignored as operational secret material.
|
||||||
|
|
||||||
|
## Known state caveat
|
||||||
|
|
||||||
|
kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
|
||||||
|
internal `kube_system_secrets` handling. This task does not claim that risk is
|
||||||
|
solved. Task 5 must verify the generated plan and state before production apply
|
||||||
|
and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
|
||||||
|
keys, and decrypted secrets are absent from committed files and unsafe state
|
||||||
|
evidence.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
terraform {
|
||||||
|
backend "s3" {
|
||||||
|
bucket = "gitea-runner-hectic-lab"
|
||||||
|
key = "gitea-runners/kube-hetzner/terraform.tfstate"
|
||||||
|
region = "fsn1"
|
||||||
|
encrypt = true
|
||||||
|
use_lockfile = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
check "remote_state_contract" {
|
||||||
|
assert {
|
||||||
|
condition = local.production_remote_state
|
||||||
|
error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
locals {
|
||||||
|
default_storage_class = "hcloud-volumes"
|
||||||
|
|
||||||
|
control_plane_nodepools = [
|
||||||
|
{
|
||||||
|
name = "control-plane"
|
||||||
|
server_type = var.control_plane_server_type
|
||||||
|
location = var.hetzner_location
|
||||||
|
labels = []
|
||||||
|
taints = []
|
||||||
|
count = 1
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
agent_nodepools = [
|
||||||
|
{
|
||||||
|
name = "runner-workers"
|
||||||
|
server_type = var.worker_server_type
|
||||||
|
location = var.hetzner_location
|
||||||
|
labels = ["node-role.hectic-lab/gitea-runner=true"]
|
||||||
|
taints = []
|
||||||
|
count = var.worker_count
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
module "kube_hetzner" {
|
||||||
|
source = "kube-hetzner/kube-hetzner/hcloud"
|
||||||
|
version = "2.19.3"
|
||||||
|
|
||||||
|
providers = {
|
||||||
|
hcloud = hcloud
|
||||||
|
}
|
||||||
|
|
||||||
|
hcloud_token = var.hcloud_token
|
||||||
|
ssh_public_key = var.ssh_public_key
|
||||||
|
ssh_private_key = var.ssh_private_key
|
||||||
|
|
||||||
|
cluster_name = var.cluster_name
|
||||||
|
base_domain = var.base_domain
|
||||||
|
|
||||||
|
# kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
|
||||||
|
# expose that expected path without outputting kubeconfig private key material.
|
||||||
|
create_kubeconfig = true
|
||||||
|
|
||||||
|
network_region = var.network_region
|
||||||
|
load_balancer_location = var.hetzner_location
|
||||||
|
control_plane_nodepools = local.control_plane_nodepools
|
||||||
|
agent_nodepools = local.agent_nodepools
|
||||||
|
|
||||||
|
# Hetzner CSI is the required StorageClass provider for runner PVCs.
|
||||||
|
disable_hetzner_csi = false
|
||||||
|
|
||||||
|
# Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
|
||||||
|
enable_longhorn = false
|
||||||
|
|
||||||
|
# Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
|
||||||
|
# raise worker_count to 5 or increase worker_server_type after validating pod
|
||||||
|
# CPU, memory, and ephemeral-storage pressure in Task 11.
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
output "kubeconfig_path" {
|
||||||
|
description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
|
||||||
|
value = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
|
||||||
|
}
|
||||||
|
|
||||||
|
output "cluster_name" {
|
||||||
|
description = "kube-hetzner cluster name."
|
||||||
|
value = var.cluster_name
|
||||||
|
}
|
||||||
|
|
||||||
|
output "node_pool_names" {
|
||||||
|
description = "Control-plane and worker node pool names used by this stack."
|
||||||
|
value = {
|
||||||
|
control_plane = [for pool in local.control_plane_nodepools : pool.name]
|
||||||
|
workers = [for pool in local.agent_nodepools : pool.name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output "default_storage_class" {
|
||||||
|
description = "Default Hetzner CSI StorageClass expected for runner PVCs."
|
||||||
|
value = local.default_storage_class
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
provider "hcloud" {}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
variable "hcloud_token" {
|
||||||
|
description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_public_key" {
|
||||||
|
description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_private_key" {
|
||||||
|
description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "cluster_name" {
|
||||||
|
description = "Name for the kube-hetzner runner cluster."
|
||||||
|
type = string
|
||||||
|
default = "gitea-runners"
|
||||||
|
|
||||||
|
validation {
|
||||||
|
condition = can(regex("^[a-z0-9-]+$", var.cluster_name))
|
||||||
|
error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "hetzner_location" {
|
||||||
|
description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
|
||||||
|
type = string
|
||||||
|
default = "fsn1"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "network_region" {
|
||||||
|
description = "Hetzner private network region. eu-central covers fsn1."
|
||||||
|
type = string
|
||||||
|
default = "eu-central"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "control_plane_server_type" {
|
||||||
|
description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
|
||||||
|
type = string
|
||||||
|
default = "cpx21"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "worker_server_type" {
|
||||||
|
description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
|
||||||
|
type = string
|
||||||
|
default = "cpx31"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "worker_count" {
|
||||||
|
description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
|
||||||
|
type = number
|
||||||
|
default = 3
|
||||||
|
|
||||||
|
validation {
|
||||||
|
condition = var.worker_count >= 1
|
||||||
|
error_message = "worker_count must be at least 1."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "kubeconfig_path" {
|
||||||
|
description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
|
||||||
|
type = string
|
||||||
|
default = null
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "base_domain" {
|
||||||
|
description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
|
||||||
|
type = string
|
||||||
|
default = ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
terraform {
|
||||||
|
required_version = ">= 1.10.1"
|
||||||
|
|
||||||
|
required_providers {
|
||||||
|
hcloud = {
|
||||||
|
source = "hetznercloud/hcloud"
|
||||||
|
version = "1.60.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
kube_hetzner_module_source = "kube-hetzner/kube-hetzner/hcloud"
|
||||||
|
kube_hetzner_module_version = "2.19.3"
|
||||||
|
hcloud_provider_minimum = ">= 1.59.0"
|
||||||
|
production_remote_state = true
|
||||||
|
}
|
||||||
@@ -0,0 +1,503 @@
|
|||||||
|
# Gitea Runner Infrastructure Runbook
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This directory is the repo-owned boundary for the first Gitea Actions runner
|
||||||
|
pool. Task 1 only establishes the scaffold and immutable decision contract;
|
||||||
|
downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
|
||||||
|
and a Nix-capable runner image under the existing subdirectories.
|
||||||
|
|
||||||
|
The target service is `https://gitea.hectic-lab.com` for the Gitea organization
|
||||||
|
`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
|
||||||
|
workflow job containers are ephemeral, while each runner pod keeps its runner
|
||||||
|
identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
|
||||||
|
|
||||||
|
## Immutable decisions
|
||||||
|
|
||||||
|
- Infrastructure is managed with OpenTofu command examples only, using the
|
||||||
|
`tofu` CLI.
|
||||||
|
- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
|
||||||
|
- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
|
||||||
|
- Runner implementation is the non-Enterprise `gitea/runner`.
|
||||||
|
- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
|
||||||
|
- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
|
||||||
|
file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
|
||||||
|
environment variables are not the contract.
|
||||||
|
- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
|
||||||
|
`/data`, including `/data/.runner`.
|
||||||
|
- Container builds run through privileged rootful DinD inside trusted runner
|
||||||
|
pods; host Docker socket mounting is not an implementation path.
|
||||||
|
- The active runner label is `ubuntu-latest`. The `nix` label is not live until
|
||||||
|
the Nix-capable image has been pushed and a concrete registry-reported digest
|
||||||
|
is added to the runner ConfigMap.
|
||||||
|
- First scope is trusted internal workflows only, with no untrusted fork or PR
|
||||||
|
workflow support.
|
||||||
|
- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
|
||||||
|
|
||||||
|
## Lifecycle boundaries
|
||||||
|
|
||||||
|
- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
|
||||||
|
contract, Hetzner provider configuration, and kube-hetzner module wiring.
|
||||||
|
- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
|
||||||
|
StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
|
||||||
|
- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
|
||||||
|
handoff; package or flake output changes are outside Task 1.
|
||||||
|
- `infra/gitea-runners/runbook.md`: this contract plus later operational
|
||||||
|
commands, rollback notes, and acceptance evidence references.
|
||||||
|
|
||||||
|
## Guardrails
|
||||||
|
|
||||||
|
- Enterprise ARC/actions-runner-controller are rejected alternatives and must
|
||||||
|
not be implemented here. Do not add ARC custom resources, controller install
|
||||||
|
instructions, or GitHub Actions ARC assumptions.
|
||||||
|
- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
|
||||||
|
acceptable for trusted internal jobs.
|
||||||
|
- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
|
||||||
|
runner pool.
|
||||||
|
- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
|
||||||
|
S3 credentials, decrypted SOPS files, or SOPS age keys.
|
||||||
|
- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
|
||||||
|
runner token; Kubernetes receives the token as a mounted file secret instead.
|
||||||
|
- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
|
||||||
|
jobs must reach the public HTTPS service.
|
||||||
|
|
||||||
|
## Initial acceptance commands
|
||||||
|
|
||||||
|
Run from the repository root:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
|
||||||
|
test -f infra/gitea-runners/runbook.md
|
||||||
|
grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
|
||||||
|
grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
|
||||||
|
grep -R "[t]erraform " infra/gitea-runners || true
|
||||||
|
grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected outcomes: the directory and file checks exit 0; the architecture-term
|
||||||
|
grep shows this contract; ARC references appear only in the rejected-alternative
|
||||||
|
guardrail above; there are no forbidden CLI command examples and no unresolved
|
||||||
|
decision placeholders.
|
||||||
|
|
||||||
|
## Downstream placeholders
|
||||||
|
|
||||||
|
- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
|
||||||
|
- Task 3: add SOPS secret contract and runtime token delivery details.
|
||||||
|
- Task 4: define or package the Nix-capable runner image for the `nix` label.
|
||||||
|
- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
|
||||||
|
and document cleanup, rollback, and scaling operations.
|
||||||
|
|
||||||
|
## Runner lifecycle cleanup
|
||||||
|
|
||||||
|
All lifecycle commands are scoped to the runner namespace:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
|
||||||
|
runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
|
||||||
|
does not delete pods, PVCs, Docker data, or Gitea runner registrations.
|
||||||
|
|
||||||
|
Run the same inventory on demand without waiting for the schedule:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||||
|
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||||
|
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
|
||||||
|
manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
|
||||||
|
operator to remove finished jobs manually.
|
||||||
|
|
||||||
|
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||||
|
pod loses `/data/.runner`. That runner identity must then be deregistered from
|
||||||
|
Gitea or the replacement pod must be allowed to re-register intentionally with
|
||||||
|
the current organization runner token. Do not delete an active runner PVC as a
|
||||||
|
normal cleanup step.
|
||||||
|
|
||||||
|
Non-UI Gitea registration reconciliation uses the Gitea API with a separate
|
||||||
|
admin token. Store that token outside this repository and pass it as a file; do
|
||||||
|
not print it:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||||
|
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||||
|
--restart=Never \
|
||||||
|
--image=curlimages/curl:8.10.1 \
|
||||||
|
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
|
||||||
|
has completed and its logs have been collected. Do not keep this admin token in
|
||||||
|
the runner namespace longer than the reconciliation window.
|
||||||
|
|
||||||
|
Only remove a stale Gitea runner registration after the corresponding pod/PVC
|
||||||
|
was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
|
||||||
|
Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
|
||||||
|
cleanup is a fallback, not the only path. Record the removed runner name and the
|
||||||
|
Kubernetes PVC/pod deletion that made it stale.
|
||||||
|
|
||||||
|
After the dry-run list identifies a stale registration and the PVC/pod deletion
|
||||||
|
has been recorded, remove that exact Gitea runner by id through the API:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||||
|
umask 077
|
||||||
|
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||||
|
trap 'rm -f "$curl_config"' EXIT
|
||||||
|
{
|
||||||
|
printf 'request = "DELETE"\n'
|
||||||
|
printf 'header = "Authorization: token '
|
||||||
|
cat /secure/path/gitea-admin-token
|
||||||
|
printf '"\n'
|
||||||
|
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||||
|
} > "$curl_config"
|
||||||
|
curl -fsS --config "$curl_config"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run the delete command for a runner that still has an active
|
||||||
|
`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
|
||||||
|
being intentionally reset for re-registration.
|
||||||
|
|
||||||
|
## Docker-in-Docker storage cleanup
|
||||||
|
|
||||||
|
Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
|
||||||
|
pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
|
||||||
|
list disk usage before pruning, and run the command only against the `docker`
|
||||||
|
container in runner pods in `gitea-runners`. Because this storage is pod-local,
|
||||||
|
loop over pods for pool-wide cleanup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
For one pod, replace the StatefulSet target with the pod name:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
|
||||||
|
kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run host-level Docker cleanup commands and do not mount or prune a host
|
||||||
|
Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
|
||||||
|
Kubernetes; the `/data` PVC remains and still controls runner identity.
|
||||||
|
|
||||||
|
## Token rotation
|
||||||
|
|
||||||
|
Rotate the Gitea organization runner token without printing decrypted values:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sops sus/gitea-runners.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
kubectl -n gitea-runners rollout restart statefulset/gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||||
|
```
|
||||||
|
|
||||||
|
The `rollout restart` command above is the controlled restart path for this
|
||||||
|
StatefulSet. Observe the rollout and each ordinal until all replacement pods are
|
||||||
|
Ready; do not delete runner pods directly as part of normal token rotation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
|
||||||
|
kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
Verification must confirm the token file mount remains present while the token
|
||||||
|
value never appears in logs or evidence:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deploy and status
|
||||||
|
|
||||||
|
These commands are executable only when the external inputs are available:
|
||||||
|
|
||||||
|
- `TF_VAR_hcloud_token`
|
||||||
|
- `TF_VAR_ssh_public_key`
|
||||||
|
- `TF_VAR_ssh_private_key`
|
||||||
|
- S3 backend credentials and endpoint access
|
||||||
|
- a matching SOPS age identity for `sus/gitea-runners.yaml`
|
||||||
|
- `kubectl` access to the target cluster
|
||||||
|
- a concrete digest for the pushed Nix-capable runner image, if enabling the
|
||||||
|
`nix` label
|
||||||
|
|
||||||
|
If any input is missing, stop before `tofu apply`. Do not guess values or reuse
|
||||||
|
stale kubeconfig files.
|
||||||
|
|
||||||
|
Before production Kubernetes apply or rollout, satisfy both manifest gates:
|
||||||
|
|
||||||
|
1. Create or update the `gitea-runner-token` Secret from SOPS. The active
|
||||||
|
Kustomize overlay intentionally does not include a placeholder Secret, but
|
||||||
|
the StatefulSet still mounts `secretName: gitea-runner-token` as
|
||||||
|
`/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
|
||||||
|
2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
|
||||||
|
image has been pushed successfully. Enable the `nix` label only by adding a
|
||||||
|
digest-pinned `docker://` mapping with the exact registry-reported sha256
|
||||||
|
digest from that push.
|
||||||
|
|
||||||
|
Use the same SOPS materialization pattern as token rotation before applying the
|
||||||
|
Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
|
||||||
|
target namespace; the full overlay remains gated on the Secret and digest
|
||||||
|
decisions:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
|
||||||
|
umask 077
|
||||||
|
token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
|
||||||
|
trap 'rm -f "$token_file"' EXIT
|
||||||
|
sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-token \
|
||||||
|
--from-file=token="$token_file" \
|
||||||
|
--dry-run=client \
|
||||||
|
-o yaml | kubectl -n gitea-runners apply -f -
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
|
||||||
|
above succeeds. Do not claim or enable the `nix` runner label until the image
|
||||||
|
publication step has produced the concrete digest.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu validate
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
|
||||||
|
export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
|
||||||
|
kubectl config current-context
|
||||||
|
kubectl get nodes -o wide
|
||||||
|
kubectl get sc
|
||||||
|
kubectl apply -k infra/gitea-runners/k8s
|
||||||
|
kubectl -n gitea-runners get statefulset gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected status after deploy:
|
||||||
|
|
||||||
|
- `kubectl config current-context` names the runner cluster context.
|
||||||
|
- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
|
||||||
|
- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
|
||||||
|
- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
|
||||||
|
- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
|
||||||
|
- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
|
||||||
|
|
||||||
|
## Scale 5 to 10 to 5
|
||||||
|
|
||||||
|
Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
|
||||||
|
wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
|
||||||
|
# Run the concurrent smoke workflows now.
|
||||||
|
|
||||||
|
kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
```
|
||||||
|
|
||||||
|
After scaling back down, inspect the cleanup dry-run and deregister any stale
|
||||||
|
runner registrations only for pods or PVCs that were intentionally removed.
|
||||||
|
|
||||||
|
## Cleanup and stale runner deregistration
|
||||||
|
|
||||||
|
Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
|
||||||
|
PVC candidates whose pod is gone. It must not delete active resources.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
|
||||||
|
kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
|
||||||
|
kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Pool-wide DinD storage checks and cleanup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
|
||||||
|
done
|
||||||
|
|
||||||
|
for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
|
||||||
|
kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
|
||||||
|
pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
|
||||||
|
replacement pod re-register intentionally with the current organization token.
|
||||||
|
Never delete an active runner PVC as routine cleanup.
|
||||||
|
|
||||||
|
Non-UI Gitea registration reconciliation uses an admin token stored outside this
|
||||||
|
repository:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
|
||||||
|
kubectl -n gitea-runners run gitea-runner-registration-dry-run \
|
||||||
|
--restart=Never \
|
||||||
|
--image=curlimages/curl:8.10.1 \
|
||||||
|
--overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
|
||||||
|
kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
|
||||||
|
pod has completed and its logs have been collected.
|
||||||
|
|
||||||
|
After the dry-run list identifies a stale registration and the PVC or pod
|
||||||
|
deletion has been recorded, remove that exact Gitea runner by id through the
|
||||||
|
API:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
runner_id='REPLACE_WITH_STALE_RUNNER_ID'
|
||||||
|
umask 077
|
||||||
|
curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
|
||||||
|
trap 'rm -f "$curl_config"' EXIT
|
||||||
|
{
|
||||||
|
printf 'request = "DELETE"\n'
|
||||||
|
printf 'header = "Authorization: token '
|
||||||
|
cat /secure/path/gitea-admin-token
|
||||||
|
printf '"\n'
|
||||||
|
printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
|
||||||
|
} > "$curl_config"
|
||||||
|
curl -fsS --config "$curl_config"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not run the delete command for a runner that still has an active
|
||||||
|
`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
|
||||||
|
being intentionally reset for re-registration.
|
||||||
|
|
||||||
|
## Application rollback
|
||||||
|
|
||||||
|
Rollback the app layer only. Do not use this section to destroy the cluster.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners rollout history statefulset/gitea-runner
|
||||||
|
kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
|
||||||
|
```
|
||||||
|
|
||||||
|
If a manifest rollback is needed, reapply the repo overlay after checking out the
|
||||||
|
known-good revision, then re-run the rollout checks:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
|
||||||
|
kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
|
||||||
|
```
|
||||||
|
|
||||||
|
## Full cluster teardown
|
||||||
|
|
||||||
|
This destroys Hetzner resources owned by the kube-hetzner stack, including the
|
||||||
|
`gitea-runners` cluster nodes, the `control-plane` node pool, the
|
||||||
|
`runner-workers` node pool, the cluster network, load balancer resources,
|
||||||
|
firewall objects, and any attached Hetzner CSI volumes still managed by the
|
||||||
|
stack. Do not run teardown unless the destruction is intentional.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
|
||||||
|
```
|
||||||
|
|
||||||
|
## Partial OpenTofu apply recovery
|
||||||
|
|
||||||
|
If `tofu apply` fails after creating some resources, do not destroy blindly.
|
||||||
|
First reconcile state and inspect what the stack thinks exists:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu state list
|
||||||
|
```
|
||||||
|
|
||||||
|
Then rerun the normal plan path. Use `-target` only as a last resort when a
|
||||||
|
single resource is stuck and the drift is understood.
|
||||||
|
|
||||||
|
## S3 backend recovery
|
||||||
|
|
||||||
|
If backend init or state access fails, first verify the bucket and versioning
|
||||||
|
outside OpenTofu, then reconfigure the backend:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
|
||||||
|
nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
|
||||||
|
tofu -chdir=infra/gitea-runners/opentofu plan
|
||||||
|
```
|
||||||
|
|
||||||
|
If the backend reports a stale lock, confirm no `tofu` process is active, then
|
||||||
|
use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
|
||||||
|
unlock a live plan or apply.
|
||||||
|
|
||||||
|
## Gitea outage troubleshooting
|
||||||
|
|
||||||
|
Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
|
||||||
|
containers.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
|
||||||
|
kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
|
||||||
|
kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
|
||||||
|
```
|
||||||
|
|
||||||
|
If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
|
||||||
|
`/data/.runner` just because the service is unavailable. Once Gitea returns,
|
||||||
|
repeat the token rotation or re-registration path if a pod restarted while the
|
||||||
|
service was unavailable and lost its runner identity.
|
||||||
|
|
||||||
|
## Release checklist
|
||||||
|
|
||||||
|
Do not release unless the following evidence files exist and are readable:
|
||||||
|
|
||||||
|
- `.sisyphus/evidence/task-5-cluster-plan.txt`
|
||||||
|
- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
|
||||||
|
- `.sisyphus/evidence/task-9-deploy.txt`
|
||||||
|
- `.sisyphus/evidence/task-9-secret-mount.txt`
|
||||||
|
- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
|
||||||
|
- `.sisyphus/evidence/task-10-nix-workflow.txt`
|
||||||
|
- `.sisyphus/evidence/task-11-scale.txt`
|
||||||
|
- `.sisyphus/evidence/task-11-restart-cleanup.txt`
|
||||||
|
|
||||||
|
If any evidence file is missing, stop and collect it before treating the runbook
|
||||||
|
as complete.
|
||||||
+71
-12
@@ -26,12 +26,39 @@
|
|||||||
"aarch64-darwin"
|
"aarch64-darwin"
|
||||||
];
|
];
|
||||||
|
|
||||||
|
cudaUnfreeNames = [
|
||||||
|
"cuda_nvcc"
|
||||||
|
"cuda_cudart"
|
||||||
|
"cuda_cuobjdump"
|
||||||
|
"cuda_cupti"
|
||||||
|
"cuda_nvdisasm"
|
||||||
|
"cuda_cccl"
|
||||||
|
"cuda_nvml_dev"
|
||||||
|
"cuda_nvrtc"
|
||||||
|
"cuda_nvtx"
|
||||||
|
"cuda_profiler_api"
|
||||||
|
|
||||||
|
"libcusparse_lt"
|
||||||
|
"libcublas"
|
||||||
|
"libcufft"
|
||||||
|
"libcufile"
|
||||||
|
"libcurand"
|
||||||
|
"libcusolver"
|
||||||
|
"libnvjitlink"
|
||||||
|
"libcusparse"
|
||||||
|
"cudnn"
|
||||||
|
];
|
||||||
|
|
||||||
|
cudaUnfreePredicate = pkg:
|
||||||
|
builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
|
||||||
|
|
||||||
forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
|
forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
|
||||||
builtins.foldl' (
|
builtins.foldl' (
|
||||||
acc: system: let
|
acc: system: let
|
||||||
pkgs = import nixpkgs {
|
pkgs = import nixpkgs {
|
||||||
inherit system;
|
inherit system;
|
||||||
overlays = pkgOverlays;
|
overlays = pkgOverlays;
|
||||||
|
config.allowUnfreePredicate = cudaUnfreePredicate;
|
||||||
};
|
};
|
||||||
systemOutputs = f {
|
systemOutputs = f {
|
||||||
system = system;
|
system = system;
|
||||||
@@ -58,7 +85,7 @@
|
|||||||
else {};
|
else {};
|
||||||
in {
|
in {
|
||||||
# -- For all systems --
|
# -- For all systems --
|
||||||
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems;
|
inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
|
||||||
|
|
||||||
forSystems = systems: nixpkgs.lib.genAttrs systems;
|
forSystems = systems: nixpkgs.lib.genAttrs systems;
|
||||||
forAllSystems = nixpkgs.lib.genAttrs AllSystems;
|
forAllSystems = nixpkgs.lib.genAttrs AllSystems;
|
||||||
@@ -67,6 +94,7 @@ in {
|
|||||||
logs = builtins.readFile ./shell/logs.sh;
|
logs = builtins.readFile ./shell/logs.sh;
|
||||||
check-tool = builtins.readFile ./shell/check-tool.sh;
|
check-tool = builtins.readFile ./shell/check-tool.sh;
|
||||||
local-dir = builtins.readFile ./shell/local-dir.sh;
|
local-dir = builtins.readFile ./shell/local-dir.sh;
|
||||||
|
load-sops = builtins.readFile ./shell/load-sops.sh;
|
||||||
};
|
};
|
||||||
|
|
||||||
sharedShellAliases = {
|
sharedShellAliases = {
|
||||||
@@ -102,7 +130,7 @@ in {
|
|||||||
|
|
||||||
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
|
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
|
||||||
# for everything that creates objects in the `hectic` namespace, used by
|
# for everything that creates objects in the `hectic` namespace, used by
|
||||||
# migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply
|
# migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
|
||||||
# the full bundle via lib/hook/apply-hectic-bundle.sh.
|
# the full bundle via lib/hook/apply-hectic-bundle.sh.
|
||||||
#
|
#
|
||||||
# The whole hectic system shares one `versionString`; `hectic-version.sql`
|
# The whole hectic system shares one `versionString`; `hectic-version.sql`
|
||||||
@@ -115,19 +143,42 @@ in {
|
|||||||
hectic = let
|
hectic = let
|
||||||
versionString = lib.fileContents ./hook/sql/HECTIC_VERSION;
|
versionString = lib.fileContents ./hook/sql/HECTIC_VERSION;
|
||||||
static = path: { inherit path; sql = builtins.readFile path; };
|
static = path: { inherit path; sql = builtins.readFile path; };
|
||||||
templated = path: {
|
templated = path: let
|
||||||
sql = builtins.replaceStrings
|
sql = builtins.replaceStrings
|
||||||
[ "@HECTIC_VERSION@" ]
|
[ "@HECTIC_VERSION@" ]
|
||||||
[ versionString ]
|
[ versionString ]
|
||||||
(builtins.readFile path);
|
(builtins.readFile path);
|
||||||
|
in {
|
||||||
|
inherit sql;
|
||||||
|
path = builtins.toFile (builtins.baseNameOf (toString path)) sql;
|
||||||
};
|
};
|
||||||
in {
|
in rec {
|
||||||
inherit versionString;
|
inherit versionString;
|
||||||
version = templated ./hook/sql/hectic-version.sql;
|
version = templated ./hook/sql/hectic-version.sql;
|
||||||
secret = static ./hook/sql/hectic-secret.sql;
|
secret = static ./hook/sql/hectic-secret.sql;
|
||||||
migration = static ./hook/sql/hectic-migration.sql;
|
migration = static ./hook/sql/hectic-migration.sql;
|
||||||
inheritance = static ./hook/sql/hectic-inheritance.sql;
|
inheritance = static ./hook/sql/hectic-inheritance.sql;
|
||||||
applyBundleScript = ./hook/apply-hectic-bundle.sh;
|
bundleFiles = [
|
||||||
|
version.path
|
||||||
|
secret.path
|
||||||
|
migration.path
|
||||||
|
inheritance.path
|
||||||
|
];
|
||||||
|
applyBundleScript =
|
||||||
|
builtins.replaceStrings
|
||||||
|
[
|
||||||
|
"@HECTIC_VERSION_SQL@"
|
||||||
|
"@HECTIC_SECRET_SQL@"
|
||||||
|
"@HECTIC_MIGRATION_SQL@"
|
||||||
|
"@HECTIC_INHERITANCE_SQL@"
|
||||||
|
]
|
||||||
|
[
|
||||||
|
"${version.path}"
|
||||||
|
"${secret.path}"
|
||||||
|
"${migration.path}"
|
||||||
|
"${inheritance.path}"
|
||||||
|
]
|
||||||
|
(builtins.readFile ./hook/apply-hectic-bundle.sh);
|
||||||
};
|
};
|
||||||
|
|
||||||
# Back-compat alias. Prefer `self.lib.hectic.inheritance`.
|
# Back-compat alias. Prefer `self.lib.hectic.inheritance`.
|
||||||
@@ -164,19 +215,27 @@ in {
|
|||||||
readModulesRecursive' = path: extraArgs:
|
readModulesRecursive' = path: extraArgs:
|
||||||
with lib;
|
with lib;
|
||||||
with builtins; let
|
with builtins; let
|
||||||
paths = pipe "${path}" [
|
collectPaths = dir: prefix:
|
||||||
(filesystem.listFilesRecursive)
|
concatLists (mapAttrsToList (name: type: let
|
||||||
(filter (hasSuffix ".nix"))
|
path' = dir + "/${name}";
|
||||||
];
|
name' = if prefix == "" then name else "${prefix}/${name}";
|
||||||
|
in
|
||||||
|
if type == "directory"
|
||||||
|
then collectPaths path' name'
|
||||||
|
else [{
|
||||||
|
inherit path';
|
||||||
|
name = name';
|
||||||
|
}]
|
||||||
|
) (readDir dir));
|
||||||
|
paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
|
||||||
pathToName = flip pipe [
|
pathToName = flip pipe [
|
||||||
(removePrefix "${path}/")
|
|
||||||
(replaceStrings ["/" ".nix"] ["." ""])
|
(replaceStrings ["/" ".nix"] ["." ""])
|
||||||
(removeSuffix ".nix")
|
(removeSuffix ".nix")
|
||||||
];
|
];
|
||||||
attrList =
|
attrList =
|
||||||
map (path': {
|
map (path': {
|
||||||
name = pathToName (unsafeDiscardStringContext path');
|
name = pathToName path'.name;
|
||||||
value = import path' extraArgs;
|
value = import path'.path' extraArgs;
|
||||||
})
|
})
|
||||||
paths;
|
paths;
|
||||||
in
|
in
|
||||||
|
|||||||
@@ -7,17 +7,12 @@
|
|||||||
#
|
#
|
||||||
# Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
|
# Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
|
||||||
#
|
#
|
||||||
# Required env (caller injects from Nix):
|
|
||||||
# HECTIC_VERSION_SQL - path to hectic-version.sql (substituted)
|
|
||||||
# HECTIC_SECRET_SQL - path to hectic-secret.sql
|
|
||||||
# HECTIC_MIGRATION_SQL - path to hectic-migration.sql
|
|
||||||
# HECTIC_INHERITANCE_SQL - path to hectic-inheritance.sql
|
|
||||||
#
|
|
||||||
# Usage:
|
# Usage:
|
||||||
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
||||||
#
|
#
|
||||||
# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via
|
# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
|
||||||
# hectic.load_secrets_from_env() after the bundle is applied.
|
# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
|
||||||
|
# SQL file paths are substituted by Nix evaluation time.
|
||||||
|
|
||||||
apply_hectic_bundle() {
|
apply_hectic_bundle() {
|
||||||
pgurl="${1:-}"
|
pgurl="${1:-}"
|
||||||
@@ -28,29 +23,27 @@ apply_hectic_bundle() {
|
|||||||
return 3
|
return 3
|
||||||
fi
|
fi
|
||||||
|
|
||||||
for var in HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL; do
|
set -- \
|
||||||
eval "val=\${$var:-}"
|
"@HECTIC_VERSION_SQL@" \
|
||||||
if [ -z "$val" ]; then
|
"@HECTIC_SECRET_SQL@" \
|
||||||
printf '%s\n' "apply-hectic-bundle: $var not set" >&2
|
"@HECTIC_MIGRATION_SQL@" \
|
||||||
return 3
|
"@HECTIC_INHERITANCE_SQL@"
|
||||||
fi
|
|
||||||
if [ ! -r "$val" ]; then
|
for sql_path do
|
||||||
printf '%s\n' "apply-hectic-bundle: $var not readable: $val" >&2
|
if [ ! -r "$sql_path" ]; then
|
||||||
|
printf '%s\n' "apply-hectic-bundle: SQL file not readable: $sql_path" >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_VERSION_SQL" || return 1
|
for sql_path do
|
||||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_SECRET_SQL" || return 1
|
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$sql_path" || return 1
|
||||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_MIGRATION_SQL" || return 1
|
done
|
||||||
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_INHERITANCE_SQL" || return 1
|
|
||||||
|
|
||||||
if [ -n "$env_content" ]; then
|
if [ -n "$env_content" ]; then
|
||||||
# Dollar-quote with $ps_env$ tag to preserve all content verbatim.
|
env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
|
||||||
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
|
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
|
||||||
SELECT hectic.load_secrets_from_env(\$ps_env\$
|
SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
|
||||||
$env_content
|
|
||||||
\$ps_env\$);
|
|
||||||
SQL
|
SQL
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
+23
-25
@@ -4,8 +4,8 @@ Single source of truth for every object created in the `hectic` PostgreSQL
|
|||||||
schema. Consumed by:
|
schema. Consumed by:
|
||||||
|
|
||||||
- `package/migrator` — applies the bundle on `migrator init` (mandatory).
|
- `package/migrator` — applies the bundle on `migrator init` (mandatory).
|
||||||
- `package/db-tool` — applies the bundle in `database hydrate` (default; opt
|
- `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
|
||||||
out with `--no-hook`).
|
(default; opt out with `--no-hook`) and in `db-ops secrets load`.
|
||||||
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
|
- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
|
||||||
paths exposed via `self.lib.hectic.*.path`.
|
paths exposed via `self.lib.hectic.*.path`.
|
||||||
|
|
||||||
@@ -42,23 +42,26 @@ that already matches.
|
|||||||
```nix
|
```nix
|
||||||
self.lib.hectic = {
|
self.lib.hectic = {
|
||||||
versionString; # e.g. "0.1.0"
|
versionString; # e.g. "0.1.0"
|
||||||
version = { sql; }; # templated
|
version = { sql; path; }; # templated
|
||||||
secret = { sql; path; };
|
secret = { sql; path; };
|
||||||
migration = { sql; path; };
|
migration = { sql; path; };
|
||||||
inheritance = { sql; path; };
|
inheritance = { sql; path; };
|
||||||
applyBundleScript; # ./hook/apply-hectic-bundle.sh
|
bundleFiles; # ordered bundle file paths
|
||||||
|
applyBundleScript; # generated helper shell source with paths embedded
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
`.sql` is the file contents as a string. `.path` is the Nix store path of the
|
`.sql` is the file contents as a string. `.path` is the Nix store path of the
|
||||||
verbatim source (only available on non-templated entries; consumers needing a
|
materialized file to pass to `psql -f`. `version.path` is generated at Nix
|
||||||
materialized version of `version.sql` must do
|
evaluation time from the templated SQL; the other `*.path` entries point at the
|
||||||
`pkgs.runCommand "hectic-version.sql" { text = self.lib.hectic.version.sql; passAsFile = ["text"]; } ''cp "$textPath" "$out"''`).
|
verbatim source files in the store.
|
||||||
|
|
||||||
## Shell helper (`apply-hectic-bundle.sh`)
|
## Shell helper (`apply-hectic-bundle.sh`)
|
||||||
|
|
||||||
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper sourced by both
|
`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
|
||||||
`migrator` and `db-tool`. Public entry point:
|
`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
|
||||||
|
SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
|
||||||
|
shell source directly into their generated scripts. Public entry point:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
||||||
@@ -70,28 +73,23 @@ apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
|
|||||||
dollar-quoted (`$ps_env$`) string so secret values cannot terminate the
|
dollar-quoted (`$ps_env$`) string so secret values cannot terminate the
|
||||||
literal.
|
literal.
|
||||||
|
|
||||||
Required environment (paths to the SQL files):
|
The SQL file paths are embedded into the helper at Nix evaluation time, so
|
||||||
|
callers only need to source the generated script and call the function.
|
||||||
- `HECTIC_VERSION_SQL`
|
External consumers that do not want to source the helper can still invoke
|
||||||
- `HECTIC_SECRET_SQL`
|
`psql -f` against `self.lib.hectic.bundleFiles` or the individual
|
||||||
- `HECTIC_MIGRATION_SQL`
|
`self.lib.hectic.*.path` entries directly.
|
||||||
- `HECTIC_INHERITANCE_SQL`
|
|
||||||
|
|
||||||
`migrator` and `db-tool` set these via Nix at build time. External consumers
|
|
||||||
typically invoke `psql -f` against the paths directly instead of sourcing the
|
|
||||||
helper.
|
|
||||||
|
|
||||||
## Adding a new SQL file
|
## Adding a new SQL file
|
||||||
|
|
||||||
1. Add `lib/hook/sql/hectic-<name>.sql`.
|
1. Add `lib/hook/sql/hectic-<name>.sql`.
|
||||||
2. Wire it into `lib/default.nix` under `lib.hectic.<name>`.
|
2. Wire it into `lib/default.nix` under `lib.hectic.<name>`.
|
||||||
3. Inject `HECTIC_<NAME>_SQL` in both `package/migrator/default.nix` and
|
3. Add its `.path` to `lib.hectic.bundleFiles` in the correct order.
|
||||||
`package/db-tool/default.nix`.
|
4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
|
||||||
4. Append a `psql -f "$HECTIC_<NAME>_SQL"` step to
|
update `lib/hook/apply-hectic-bundle.sh` to apply the file.
|
||||||
`lib/hook/apply-hectic-bundle.sh` in the correct order.
|
|
||||||
5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
|
5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
|
||||||
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`
|
6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
|
||||||
and `test/package/db-tool/test/postgresql/hydrate-hook/`.
|
`test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
|
||||||
|
bundle-loading coverage.
|
||||||
|
|
||||||
## Versioning
|
## Versioning
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
load_sops_shell_quote() {
|
||||||
|
printf "'"
|
||||||
|
printf '%s' "$1" | sed "s/'/'\"'\"'/g"
|
||||||
|
printf "'"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_normalize_key() {
|
||||||
|
load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
|
||||||
|
|
||||||
|
case "$load_sops_normalized_key" in
|
||||||
|
''|[!A-Z_]*|*[!A-Z0-9_]*)
|
||||||
|
printf 'load-sops: invalid environment name after key normalization\n' >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
printf '%s\n' "$load_sops_normalized_key"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_env_from_sops_file() {
|
||||||
|
load_sops_file=$1
|
||||||
|
load_sops_extract=${2-}
|
||||||
|
|
||||||
|
if ! command -v sops >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `sops` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v yq >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `yq` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_decrypted=''
|
||||||
|
load_sops_attempt=0
|
||||||
|
load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
|
||||||
|
load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
|
||||||
|
|
||||||
|
while :; do
|
||||||
|
if [ -n "$load_sops_extract" ]; then
|
||||||
|
if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
|
||||||
|
load_sops_status=0
|
||||||
|
else
|
||||||
|
load_sops_status=$?
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
|
||||||
|
load_sops_status=0
|
||||||
|
else
|
||||||
|
load_sops_status=$?
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_status" -eq 0 ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_prompt" != 1 ]; then
|
||||||
|
printf 'load-sops: failed to decrypt file\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
|
||||||
|
printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_attempt=$((load_sops_attempt + 1))
|
||||||
|
if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
|
||||||
|
printf 'load-sops: decrypt failed after configured retries\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
|
||||||
|
load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
|
||||||
|
load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
|
||||||
|
case "$load_sops_use_script" in
|
||||||
|
auto)
|
||||||
|
if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
|
||||||
|
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
|
||||||
|
load_sops_script_status=0
|
||||||
|
else
|
||||||
|
load_sops_script_status=$?
|
||||||
|
fi
|
||||||
|
if [ "$load_sops_script_status" -eq 0 ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
1)
|
||||||
|
if ! command -v script >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `script` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
|
||||||
|
load_sops_script_status=0
|
||||||
|
else
|
||||||
|
load_sops_script_status=$?
|
||||||
|
fi
|
||||||
|
if [ "$load_sops_script_status" -eq 0 ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
0)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
|
||||||
|
if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
|
||||||
|
printf 'load-sops: failed to read prompt input\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
export SOPS_AGE_KEY_CMD
|
||||||
|
done
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_text "$load_sops_decrypted"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_file() {
|
||||||
|
load_sops_file=$1
|
||||||
|
|
||||||
|
if ! command -v yq >/dev/null 2>&1; then
|
||||||
|
printf 'load-sops: required tool `yq` not found\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sops_env_from_yaml_text() {
|
||||||
|
load_sops_yaml=$1
|
||||||
|
load_sops_seen=''
|
||||||
|
|
||||||
|
if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
|
||||||
|
load_sops_keys_status=0
|
||||||
|
else
|
||||||
|
load_sops_keys_status=$?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_keys_status" -ne 0 ]; then
|
||||||
|
printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS= read -r load_sops_key; do
|
||||||
|
[ -n "$load_sops_key" ] || continue
|
||||||
|
|
||||||
|
load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
|
||||||
|
|
||||||
|
case "
|
||||||
|
$load_sops_seen
|
||||||
|
" in
|
||||||
|
*"
|
||||||
|
$load_sops_name
|
||||||
|
"*)
|
||||||
|
if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
|
||||||
|
printf 'load-sops: normalized environment name collision\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
load_sops_seen=${load_sops_seen}${load_sops_seen:+"
|
||||||
|
"}$load_sops_name
|
||||||
|
|
||||||
|
load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
|
||||||
|
printf 'load-sops: failed to inspect YAML value kind\n' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
|
||||||
|
printf 'load-sops: failed to inspect YAML value tag\n' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$load_sops_kind" != scalar ]; then
|
||||||
|
printf 'load-sops: top-level YAML values must be scalars\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$load_sops_tag" = '!!null' ]; then
|
||||||
|
printf 'load-sops: top-level YAML null values are not supported\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
|
||||||
|
eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
|
||||||
|
if [ -n "$load_sops_already_set" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
|
||||||
|
printf 'load-sops: failed to read YAML scalar value\n' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
|
||||||
|
eval "export $load_sops_name=$load_sops_quoted"
|
||||||
|
done <<EOF
|
||||||
|
$load_sops_keys
|
||||||
|
EOF
|
||||||
|
}
|
||||||
@@ -9,11 +9,11 @@ with self.lib;
|
|||||||
let
|
let
|
||||||
# Combine hectic modules into one
|
# Combine hectic modules into one
|
||||||
hectic.imports = attrValues (
|
hectic.imports = attrValues (
|
||||||
readModulesRecursive' ./hectic { inherit flake self inputs; }
|
readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
|
||||||
);
|
);
|
||||||
# Read generic modules separately
|
# Read generic modules separately
|
||||||
generic = readModulesRecursive'
|
generic = readModulesRecursive'
|
||||||
./generic
|
(flake + "/nixos/module/generic")
|
||||||
{ inherit flake self inputs; };
|
{ inherit flake self inputs; };
|
||||||
in generic // {
|
in generic // {
|
||||||
inherit hectic;
|
inherit hectic;
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
];
|
];
|
||||||
|
|
||||||
adminNames = [ "yukkop" ];
|
adminNames = [ "yukkop" ];
|
||||||
|
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||||
|
|
||||||
cfg = config.hectic.generic.matrix-cluster;
|
cfg = config.hectic.generic.matrix-cluster;
|
||||||
in {
|
in {
|
||||||
@@ -39,7 +40,7 @@ in {
|
|||||||
value = {
|
value = {
|
||||||
key = "matrix/users/${name}/password";
|
key = "matrix/users/${name}/password";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
}) userNames
|
}) userNames
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -250,6 +250,7 @@ in {
|
|||||||
# failover flip does not need a separate provisioning step.
|
# failover flip does not need a separate provisioning step.
|
||||||
systemd.tmpfiles.rules = [
|
systemd.tmpfiles.rules = [
|
||||||
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
|
"d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
|
||||||
|
"Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
|
||||||
];
|
];
|
||||||
|
|
||||||
systemd.services.matrix-cluster-signing-key = {
|
systemd.services.matrix-cluster-signing-key = {
|
||||||
@@ -319,6 +320,12 @@ in {
|
|||||||
media_store_path = s3Cfg.mediaStorePath;
|
media_store_path = s3Cfg.mediaStorePath;
|
||||||
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
|
signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
|
||||||
|
|
||||||
|
# Tolerate bursty Element/iPhone presence syncs without disabling limits.
|
||||||
|
rc_presence.per_user = {
|
||||||
|
per_second = 0.5;
|
||||||
|
burst_count = 5;
|
||||||
|
};
|
||||||
|
|
||||||
experimental_features = {
|
experimental_features = {
|
||||||
msc3266_enabled = true;
|
msc3266_enabled = true;
|
||||||
msc4140_enabled = true;
|
msc4140_enabled = true;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
inputs,
|
inputs,
|
||||||
flake,
|
|
||||||
self,
|
self,
|
||||||
|
...
|
||||||
}: {
|
}: {
|
||||||
pkgs,
|
pkgs,
|
||||||
lib,
|
lib,
|
||||||
@@ -27,7 +27,15 @@ in {
|
|||||||
users.defaultUserShell = pkgs.zsh;
|
users.defaultUserShell = pkgs.zsh;
|
||||||
|
|
||||||
# Enable flakes and new 'nix' command
|
# Enable flakes and new 'nix' command
|
||||||
nix.settings.experimental-features = "nix-command flakes";
|
nix.settings = {
|
||||||
|
experimental-features = "nix-command flakes";
|
||||||
|
extra-substituters = [
|
||||||
|
"https://cache.hectic-lab.com/hectic"
|
||||||
|
];
|
||||||
|
extra-trusted-public-keys = [
|
||||||
|
"hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
networking.firewall.enable = true;
|
networking.firewall.enable = true;
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
{
|
{
|
||||||
inputs,
|
...
|
||||||
flake,
|
|
||||||
self,
|
|
||||||
}:
|
}:
|
||||||
{
|
{
|
||||||
pkgs,
|
pkgs,
|
||||||
@@ -11,6 +9,13 @@
|
|||||||
}: let
|
}: let
|
||||||
cfg = config.hectic.hardware.hetzner-cloud;
|
cfg = config.hectic.hardware.hetzner-cloud;
|
||||||
isNewer = cfg.generation == "newer";
|
isNewer = cfg.generation == "newer";
|
||||||
|
networkMatchConfig =
|
||||||
|
(lib.optionalAttrs (cfg.networkMatchConfigName != null) {
|
||||||
|
Name = cfg.networkMatchConfigName;
|
||||||
|
})
|
||||||
|
// (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
|
||||||
|
PermanentMACAddress = cfg.networkMatchConfigMac;
|
||||||
|
});
|
||||||
in {
|
in {
|
||||||
options.hectic.hardware.hetzner-cloud = {
|
options.hectic.hardware.hetzner-cloud = {
|
||||||
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
|
enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
|
||||||
@@ -51,6 +56,14 @@ in {
|
|||||||
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
floatingIpv4 = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
|
||||||
|
default = null;
|
||||||
|
example = "188.243.124.247";
|
||||||
|
description = ''
|
||||||
|
Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
|
||||||
|
'';
|
||||||
|
};
|
||||||
device = lib.mkOption {
|
device = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
|
default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
|
||||||
@@ -65,14 +78,27 @@ in {
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
networkMatchConfigName = lib.mkOption {
|
networkMatchConfigName = lib.mkOption {
|
||||||
type = lib.types.strMatching "^(enp1s0|ens3|eth0)$";
|
type = with lib.types; nullOr str;
|
||||||
|
default = null;
|
||||||
example = "enp1s0";
|
example = "enp1s0";
|
||||||
description = ''
|
description = ''
|
||||||
type of network conection,
|
Optional interface name to match in systemd-networkd.
|
||||||
on older hetzner servers may be `ens3` or else
|
|
||||||
on newer probably `enp1s0`
|
|
||||||
|
|
||||||
you can use `networkctl list` on server to know it
|
Prefer `networkMatchConfigMac` for stable matching across rescue
|
||||||
|
images and installed systems that may rename interfaces differently.
|
||||||
|
|
||||||
|
You can use `networkctl list` on server to know it.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networkMatchConfigMac = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
|
||||||
|
default = null;
|
||||||
|
example = "92:00:08:4a:b0:32";
|
||||||
|
description = ''
|
||||||
|
Optional permanent MAC address to match in systemd-networkd.
|
||||||
|
|
||||||
|
This is the preferred Hetzner Cloud matching method because interface
|
||||||
|
names can differ between rescue images and installed NixOS systems.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -80,6 +106,15 @@ in {
|
|||||||
config = lib.mkIf cfg.enable (lib.mkMerge
|
config = lib.mkIf cfg.enable (lib.mkMerge
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
|
boot.loader.systemd-boot.enable = false;
|
||||||
|
boot.loader.efi.canTouchEfiVariables = false;
|
||||||
|
boot.loader.grub = {
|
||||||
|
enable = true;
|
||||||
|
efiSupport = true;
|
||||||
|
efiInstallAsRemovable = true;
|
||||||
|
device = "nodev";
|
||||||
|
};
|
||||||
|
|
||||||
boot.initrd.availableKernelModules = [
|
boot.initrd.availableKernelModules = [
|
||||||
"ata_piix"
|
"ata_piix"
|
||||||
"uhci_hcd"
|
"uhci_hcd"
|
||||||
@@ -90,12 +125,12 @@ in {
|
|||||||
networking.useNetworkd = true;
|
networking.useNetworkd = true;
|
||||||
systemd.network.enable = true;
|
systemd.network.enable = true;
|
||||||
systemd.network.networks."30-wan" = {
|
systemd.network.networks."30-wan" = {
|
||||||
matchConfig.Name = cfg.networkMatchConfigName;
|
matchConfig = networkMatchConfig;
|
||||||
networkConfig.DHCP = "no";
|
networkConfig.DHCP = "no";
|
||||||
address = [
|
address = [
|
||||||
"${cfg.ipv4}/32"
|
"${cfg.ipv4}/32"
|
||||||
"${cfg.ipv6}::/64"
|
"${cfg.ipv6}::/64"
|
||||||
];
|
] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
|
||||||
routes = [
|
routes = [
|
||||||
{ Gateway = "172.31.1.1"; GatewayOnLink = true; }
|
{ Gateway = "172.31.1.1"; GatewayOnLink = true; }
|
||||||
{ Gateway = "fe80::1"; }
|
{ Gateway = "fe80::1"; }
|
||||||
@@ -137,6 +172,13 @@ in {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
|
||||||
|
message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
|
||||||
|
}
|
||||||
|
];
|
||||||
}
|
}
|
||||||
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
|
(lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
|
||||||
boot.initrd.kernelModules = [ "virtio_gpu" ];
|
boot.initrd.kernelModules = [ "virtio_gpu" ];
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
{ ... }:
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
cfg = config.hectic.hardware.njalla;
|
||||||
|
in {
|
||||||
|
options.hectic.hardware.njalla = {
|
||||||
|
enable = lib.mkEnableOption "Enable njalla hardware configurations";
|
||||||
|
ipv4 = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
|
||||||
|
example = "185.193.126.103";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv4 address assigned to the host.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv4PrefixLength = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
default = 24;
|
||||||
|
example = 24;
|
||||||
|
description = ''
|
||||||
|
Njalla IPv4 prefix length.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv4Gateway = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
|
||||||
|
default = "185.193.126.1";
|
||||||
|
example = "185.193.126.1";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv4 gateway.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv6 = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
|
||||||
|
example = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv6 address assigned to the host.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv6PrefixLength = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
default = 64;
|
||||||
|
example = 64;
|
||||||
|
description = ''
|
||||||
|
Njalla IPv6 prefix length.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
ipv6Gateway = lib.mkOption {
|
||||||
|
type = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
|
||||||
|
default = "2a0a:3840:1337:126::1";
|
||||||
|
example = "2a0a:3840:1337:126::1";
|
||||||
|
description = ''
|
||||||
|
Njalla IPv6 gateway.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networkMatchConfigName = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "eth0";
|
||||||
|
example = "eth0";
|
||||||
|
description = ''
|
||||||
|
Njalla container network interface name.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
device = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "/dev/vda";
|
||||||
|
example = "/dev/disk/by-id/virtio-root";
|
||||||
|
description = ''
|
||||||
|
Njalla installation disk for disko/nixos-anywhere.
|
||||||
|
|
||||||
|
`/dev/vda` is the default block device visible on the inspected Njalla
|
||||||
|
host. Prefer a stable `/dev/disk/by-id/...` path when available.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
enableDisko = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = ''
|
||||||
|
Whether to provide a disko layout for nixos-anywhere installs.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable (lib.mkMerge [
|
||||||
|
{
|
||||||
|
boot.isContainer = true;
|
||||||
|
|
||||||
|
networking.useDHCP = false;
|
||||||
|
networking.useNetworkd = true;
|
||||||
|
systemd.network.enable = true;
|
||||||
|
systemd.network.networks."30-wan" = {
|
||||||
|
matchConfig.Name = cfg.networkMatchConfigName;
|
||||||
|
networkConfig.DHCP = "no";
|
||||||
|
address = [
|
||||||
|
"${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
|
||||||
|
"${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
|
||||||
|
];
|
||||||
|
routes = [
|
||||||
|
{ Gateway = cfg.ipv4Gateway; }
|
||||||
|
{ Gateway = cfg.ipv6Gateway; }
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
|
||||||
|
}
|
||||||
|
(lib.mkIf cfg.enableDisko {
|
||||||
|
boot.loader.grub.device = cfg.device;
|
||||||
|
|
||||||
|
disko.devices.disk.main = {
|
||||||
|
type = "disk";
|
||||||
|
device = cfg.device;
|
||||||
|
content = {
|
||||||
|
type = "gpt";
|
||||||
|
partitions = {
|
||||||
|
boot = {
|
||||||
|
size = "1M";
|
||||||
|
type = "EF02";
|
||||||
|
priority = 1;
|
||||||
|
};
|
||||||
|
root = {
|
||||||
|
size = "100%";
|
||||||
|
content = {
|
||||||
|
type = "filesystem";
|
||||||
|
format = "ext4";
|
||||||
|
mountpoint = "/";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
}
|
||||||
@@ -25,36 +25,7 @@ in {
|
|||||||
programs.bash.shellAliases.tmux = "tmux a";
|
programs.bash.shellAliases.tmux = "tmux a";
|
||||||
|
|
||||||
home-manager.sharedModules = [
|
home-manager.sharedModules = [
|
||||||
{
|
(flake + "/home/module/program/tmux.nix")
|
||||||
programs.tmux = {
|
|
||||||
enable = true;
|
|
||||||
plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
|
|
||||||
keyMode = "vi";
|
|
||||||
escapeTime = 500;
|
|
||||||
historyLimit = 50000;
|
|
||||||
newSession = true;
|
|
||||||
extraConfig = ''
|
|
||||||
# resurrect
|
|
||||||
set -g @resurrect-strategy-vim 'session'
|
|
||||||
set -g @resurrect-strategy-nvim 'session'
|
|
||||||
set -g @resurrect-capture-pane-contents 'on'
|
|
||||||
|
|
||||||
resurrect_dir="$HOME/.tmux/resurrect"
|
|
||||||
set -g @resurrect-dir $resurrect_dir
|
|
||||||
set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
|
|
||||||
|
|
||||||
# continuum
|
|
||||||
set -g @continuum-restore 'on'
|
|
||||||
set -g @continuum-boot 'on'
|
|
||||||
set -g @continuum-save-interval '10'
|
|
||||||
|
|
||||||
bind-key -T copy-mode-vi v send-keys -X begin-selection
|
|
||||||
bind-key -T copy-mode-vi C-v send-keys -X rectangle-toggle
|
|
||||||
|
|
||||||
bind-key O select-pane -t :.-
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
];
|
||||||
|
|
||||||
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
|
home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
{ ... }: {
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
cfg = config.hectic.services.attic;
|
||||||
|
in {
|
||||||
|
options.hectic.services.attic = {
|
||||||
|
enable = lib.mkEnableOption "Attic binary cache server";
|
||||||
|
|
||||||
|
hostName = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Public hostname used by clients to reach this Attic server.";
|
||||||
|
};
|
||||||
|
|
||||||
|
listenAddress = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "127.0.0.1";
|
||||||
|
description = "Local address atticd binds to behind the reverse proxy.";
|
||||||
|
};
|
||||||
|
|
||||||
|
port = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 8080;
|
||||||
|
description = "Local port atticd binds to behind the reverse proxy.";
|
||||||
|
};
|
||||||
|
|
||||||
|
environmentFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = ''
|
||||||
|
SOPS-backed environment file containing Attic JWT and object-storage
|
||||||
|
credentials.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
storage = {
|
||||||
|
bucket = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Hetzner Object Storage bucket name used by Attic.";
|
||||||
|
};
|
||||||
|
|
||||||
|
endpoint = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "S3-compatible HTTPS endpoint for Hetzner Object Storage.";
|
||||||
|
};
|
||||||
|
|
||||||
|
region = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Region name for Hetzner Object Storage.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable {
|
||||||
|
services.atticd = {
|
||||||
|
enable = true;
|
||||||
|
environmentFile = cfg.environmentFile;
|
||||||
|
settings = {
|
||||||
|
listen = "${cfg.listenAddress}:${toString cfg.port}";
|
||||||
|
allowed-hosts = [ cfg.hostName ];
|
||||||
|
api-endpoint = "https://${cfg.hostName}/";
|
||||||
|
compression.type = "zstd";
|
||||||
|
storage = {
|
||||||
|
type = "s3";
|
||||||
|
bucket = cfg.storage.bucket;
|
||||||
|
endpoint = cfg.storage.endpoint;
|
||||||
|
region = cfg.storage.region;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -31,7 +31,7 @@ in {
|
|||||||
|
|
||||||
locations."= /config.element.${matrixDomain}.json".return = "302 /config.json";
|
locations."= /config.element.${matrixDomain}.json".return = "302 /config.json";
|
||||||
|
|
||||||
root = pkgs.element-web.override {
|
root = pkgs.hectic.element-web.override {
|
||||||
conf = {
|
conf = {
|
||||||
default_server_config = {
|
default_server_config = {
|
||||||
"m.homeserver".base_url = "https://${matrixDomain}";
|
"m.homeserver".base_url = "https://${matrixDomain}";
|
||||||
@@ -43,6 +43,8 @@ in {
|
|||||||
matrixDomain
|
matrixDomain
|
||||||
];
|
];
|
||||||
|
|
||||||
|
hectic.videoMessages.enabled = true;
|
||||||
|
|
||||||
jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) {
|
jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) {
|
||||||
preferred_domain = jitsiPreferredDomain;
|
preferred_domain = jitsiPreferredDomain;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,224 @@
|
|||||||
|
{ ... }: {
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
cfg = config.hectic.services.ente;
|
||||||
|
|
||||||
|
webHostNames = [
|
||||||
|
cfg.domains.accounts
|
||||||
|
cfg.domains.cast
|
||||||
|
cfg.domains.photos
|
||||||
|
];
|
||||||
|
in {
|
||||||
|
options.hectic.services.ente = {
|
||||||
|
enable = lib.mkEnableOption "Ente Photos self-hosted service";
|
||||||
|
|
||||||
|
apiDomain = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Public hostname for the Ente Museum API.";
|
||||||
|
};
|
||||||
|
|
||||||
|
domains = {
|
||||||
|
accounts = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Public hostname for the Ente accounts web app.";
|
||||||
|
};
|
||||||
|
|
||||||
|
cast = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Public hostname for the Ente cast web app.";
|
||||||
|
};
|
||||||
|
|
||||||
|
albums = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Public hostname for public Ente album links.";
|
||||||
|
};
|
||||||
|
|
||||||
|
photos = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "Public hostname for the Ente Photos web app.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
maxUploadSize = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "10G";
|
||||||
|
description = "Maximum request body accepted by nginx in front of Museum.";
|
||||||
|
};
|
||||||
|
|
||||||
|
disableRegistration = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Whether Museum should reject new account registration.";
|
||||||
|
};
|
||||||
|
|
||||||
|
smtp = {
|
||||||
|
enable = lib.mkEnableOption "SMTP delivery for Ente verification emails";
|
||||||
|
|
||||||
|
host = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "127.0.0.1";
|
||||||
|
description = "SMTP host Museum uses to send verification emails.";
|
||||||
|
};
|
||||||
|
|
||||||
|
port = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 25;
|
||||||
|
description = "SMTP port Museum uses to send verification emails.";
|
||||||
|
};
|
||||||
|
|
||||||
|
email = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "From email address used by Museum.";
|
||||||
|
};
|
||||||
|
|
||||||
|
senderName = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "Ente Photos";
|
||||||
|
description = "Display name used for Ente verification emails.";
|
||||||
|
};
|
||||||
|
|
||||||
|
encryption = lib.mkOption {
|
||||||
|
type = lib.types.nullOr (lib.types.enum [ "tls" "ssl" ]);
|
||||||
|
default = null;
|
||||||
|
description = "Optional SMTP encryption mode. Leave null for local plaintext SMTP.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
storage = {
|
||||||
|
bucket = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "S3-compatible bucket used by Ente for photo object storage.";
|
||||||
|
};
|
||||||
|
|
||||||
|
endpoint = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "S3-compatible endpoint URL.";
|
||||||
|
};
|
||||||
|
|
||||||
|
region = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
description = "S3-compatible region name.";
|
||||||
|
};
|
||||||
|
|
||||||
|
hotStorage = lib.mkOption {
|
||||||
|
type = lib.types.enum [
|
||||||
|
"b2-eu-cen"
|
||||||
|
"wasabi-eu-central-2-v3"
|
||||||
|
"scw-eu-fr-v3"
|
||||||
|
];
|
||||||
|
default = "b2-eu-cen";
|
||||||
|
description = ''
|
||||||
|
Museum's primary hot-storage key. Upstream requires one of its
|
||||||
|
historical S3 storage identifiers even when the backing provider is a
|
||||||
|
generic S3-compatible service.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
usePathStyleUrls = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Whether Museum should use path-style S3 URLs.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
secrets = {
|
||||||
|
encryptionKeyFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = "File containing Museum key.encryption.";
|
||||||
|
};
|
||||||
|
|
||||||
|
hashKeyFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = "File containing Museum key.hash.";
|
||||||
|
};
|
||||||
|
|
||||||
|
jwtSecretFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = "File containing Museum jwt.secret.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3AccessKeyFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = "File containing the S3 access key.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3SecretKeyFile = lib.mkOption {
|
||||||
|
type = lib.types.path;
|
||||||
|
description = "File containing the S3 secret key.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable {
|
||||||
|
services.ente = {
|
||||||
|
api = {
|
||||||
|
enable = true;
|
||||||
|
enableLocalDB = true;
|
||||||
|
domain = cfg.apiDomain;
|
||||||
|
|
||||||
|
nginx.enable = true;
|
||||||
|
|
||||||
|
settings = {
|
||||||
|
key = {
|
||||||
|
encryption._secret = cfg.secrets.encryptionKeyFile;
|
||||||
|
hash._secret = cfg.secrets.hashKeyFile;
|
||||||
|
};
|
||||||
|
|
||||||
|
jwt.secret._secret = cfg.secrets.jwtSecretFile;
|
||||||
|
|
||||||
|
s3 = {
|
||||||
|
hot_storage.primary = cfg.storage.hotStorage;
|
||||||
|
derived-storage = cfg.storage.hotStorage;
|
||||||
|
are_local_buckets = false;
|
||||||
|
use_path_style_urls = cfg.storage.usePathStyleUrls;
|
||||||
|
|
||||||
|
${cfg.storage.hotStorage} = {
|
||||||
|
key._secret = cfg.secrets.s3AccessKeyFile;
|
||||||
|
secret._secret = cfg.secrets.s3SecretKeyFile;
|
||||||
|
endpoint = cfg.storage.endpoint;
|
||||||
|
region = cfg.storage.region;
|
||||||
|
bucket = cfg.storage.bucket;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
internal.disable-registration = cfg.disableRegistration;
|
||||||
|
|
||||||
|
smtp = lib.mkIf cfg.smtp.enable ({
|
||||||
|
inherit (cfg.smtp) host port email;
|
||||||
|
sender-name = cfg.smtp.senderName;
|
||||||
|
} // lib.optionalAttrs (cfg.smtp.encryption != null) {
|
||||||
|
encryption = cfg.smtp.encryption;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
web = {
|
||||||
|
enable = true;
|
||||||
|
domains = {
|
||||||
|
api = cfg.apiDomain;
|
||||||
|
inherit (cfg.domains) accounts cast albums photos;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
services.nginx.virtualHosts =
|
||||||
|
(lib.genAttrs webHostNames (_: {
|
||||||
|
enableACME = true;
|
||||||
|
forceSSL = true;
|
||||||
|
})) // {
|
||||||
|
${cfg.apiDomain} = {
|
||||||
|
enableACME = true;
|
||||||
|
forceSSL = true;
|
||||||
|
extraConfig = lib.mkForce ''
|
||||||
|
client_max_body_size ${cfg.maxUploadSize};
|
||||||
|
'';
|
||||||
|
locations."/".extraConfig = ''
|
||||||
|
proxy_read_timeout 600s;
|
||||||
|
proxy_send_timeout 600s;
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -56,8 +56,18 @@ in {
|
|||||||
certificateScheme = "acme-nginx";
|
certificateScheme = "acme-nginx";
|
||||||
};
|
};
|
||||||
|
|
||||||
# NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records
|
services.postfix.settings.main = {
|
||||||
services.postfix.settings.main.inet_protocols = lib.mkDefault "ipv4";
|
# NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records.
|
||||||
|
inet_protocols = lib.mkDefault "ipv4";
|
||||||
|
|
||||||
|
# NOTE(yukkop): nixos-mailserver enables DANE by default. Some large MXes
|
||||||
|
# currently fail certificate verification under this policy, which leaves
|
||||||
|
# otherwise valid transactional mail deferred in the queue. Keep STARTTLS
|
||||||
|
# opportunistic for outbound delivery rather than blocking mail entirely.
|
||||||
|
smtp_tls_security_level = lib.mkForce "may";
|
||||||
|
smtp_dns_support_level = lib.mkForce "enabled";
|
||||||
|
smtp_tls_policy_maps = lib.mkForce "";
|
||||||
|
};
|
||||||
|
|
||||||
security.acme.acceptTerms = true;
|
security.acme.acceptTerms = true;
|
||||||
security.acme.defaults.email = "security@" + cfg.domain;
|
security.acme.defaults.email = "security@" + cfg.domain;
|
||||||
|
|||||||
@@ -147,7 +147,7 @@ in {
|
|||||||
|
|
||||||
(lib.mkIf cfg.watcher.enable {
|
(lib.mkIf cfg.watcher.enable {
|
||||||
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
|
sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
|
||||||
sopsFile = "${flake}/sus/sentinella-default.yaml";
|
sopsFile = flake + "/sus/sentinella-default.yaml";
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.services."sentinella-watcher" = {
|
systemd.services."sentinella-watcher" = {
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
{ ... }: {
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
cfg = config.hectic.services.stable-video-diffusion;
|
||||||
|
in {
|
||||||
|
options.hectic.services.stable-video-diffusion = {
|
||||||
|
enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
|
||||||
|
|
||||||
|
host = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "127.0.0.1";
|
||||||
|
description = "Address the Stable Video Diffusion API binds to.";
|
||||||
|
};
|
||||||
|
|
||||||
|
port = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 7861;
|
||||||
|
description = "Port the Stable Video Diffusion API binds to.";
|
||||||
|
};
|
||||||
|
|
||||||
|
package = lib.mkOption {
|
||||||
|
type = lib.types.package;
|
||||||
|
default = pkgs.hectic.stable-video-diffusion-api;
|
||||||
|
defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
|
||||||
|
description = "Package providing the Stable Video Diffusion API executable.";
|
||||||
|
};
|
||||||
|
|
||||||
|
modelId = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "stabilityai/stable-video-diffusion-img2vid-xt";
|
||||||
|
description = "Model identifier loaded by the Stable Video Diffusion API.";
|
||||||
|
};
|
||||||
|
|
||||||
|
device = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
|
||||||
|
default = null;
|
||||||
|
description = ''
|
||||||
|
Torch device requested from the Stable Video Diffusion API. When null,
|
||||||
|
the package keeps its own auto-detection behavior.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
libraryPath = lib.mkOption {
|
||||||
|
type = with lib.types; listOf str;
|
||||||
|
default = [];
|
||||||
|
description = ''
|
||||||
|
Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
|
||||||
|
should include /run/opengl-driver/lib so libcuda.so is visible to torch.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
stateDir = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "/var/lib/stable-video-diffusion-api";
|
||||||
|
description = "Persistent state directory for the Stable Video Diffusion API.";
|
||||||
|
};
|
||||||
|
|
||||||
|
cacheDir = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "/var/cache/stable-video-diffusion-api";
|
||||||
|
description = "Cache directory for downloaded model and runtime artifacts.";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputDir = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "${cfg.stateDir}/outputs";
|
||||||
|
defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
|
||||||
|
description = "Directory where generated video outputs are written.";
|
||||||
|
};
|
||||||
|
|
||||||
|
environmentFile = lib.mkOption {
|
||||||
|
type = with lib.types; nullOr path;
|
||||||
|
default = null;
|
||||||
|
description = ''
|
||||||
|
Optional environment file for secrets or runtime overrides. Values from
|
||||||
|
the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
|
||||||
|
SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
|
||||||
|
and LD_LIBRARY_PATH by default.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
openFirewall = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Whether to open the API port in the firewall.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable {
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = cfg.device != "cuda" || cfg.libraryPath != [];
|
||||||
|
message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
users.users.stable-video-diffusion = {
|
||||||
|
isSystemUser = true;
|
||||||
|
group = "stable-video-diffusion";
|
||||||
|
};
|
||||||
|
users.groups.stable-video-diffusion = {};
|
||||||
|
|
||||||
|
systemd.tmpfiles.rules = [
|
||||||
|
"d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||||
|
"d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||||
|
"d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
|
||||||
|
];
|
||||||
|
|
||||||
|
systemd.services.stable-video-diffusion-api = {
|
||||||
|
description = "Stable Video Diffusion HTTP API";
|
||||||
|
after = [ "network.target" ];
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
environment = {
|
||||||
|
SVD_API_HOST = cfg.host;
|
||||||
|
SVD_API_PORT = toString cfg.port;
|
||||||
|
SVD_MODEL_ID = cfg.modelId;
|
||||||
|
SVD_STATE_DIR = cfg.stateDir;
|
||||||
|
SVD_CACHE_DIR = cfg.cacheDir;
|
||||||
|
SVD_OUTPUT_DIR = cfg.outputDir;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.device != null) {
|
||||||
|
SVD_DEVICE = cfg.device;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.libraryPath != []) {
|
||||||
|
LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
|
||||||
|
};
|
||||||
|
serviceConfig = lib.mkMerge [
|
||||||
|
{
|
||||||
|
Type = "simple";
|
||||||
|
User = "stable-video-diffusion";
|
||||||
|
Group = "stable-video-diffusion";
|
||||||
|
WorkingDirectory = cfg.stateDir;
|
||||||
|
ExecStart = lib.getExe' cfg.package "stable-video-diffusion-api";
|
||||||
|
Restart = "on-failure";
|
||||||
|
RestartSec = "5s";
|
||||||
|
TimeoutStopSec = "30s";
|
||||||
|
KillSignal = "SIGTERM";
|
||||||
|
KillMode = "mixed";
|
||||||
|
StandardOutput = "journal";
|
||||||
|
StandardError = "journal";
|
||||||
|
}
|
||||||
|
(lib.mkIf (cfg.environmentFile != null) {
|
||||||
|
EnvironmentFile = cfg.environmentFile;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,230 +0,0 @@
|
|||||||
# Matrix Cluster Failover Runbook (`accord.tube`)
|
|
||||||
|
|
||||||
Primary: `hectic-lab` (NL, `128.140.75.58`)
|
|
||||||
Standby: `bfs.poland.xray` (PL, `91.198.166.181`)
|
|
||||||
|
|
||||||
Module: `hectic.generic.matrix-cluster` (`nixos/module/generic/matrix-cluster.nix`).
|
|
||||||
Shared secrets: `sus/matrix-cluster.yaml`.
|
|
||||||
|
|
||||||
All `psql` and `pg_ctl` invocations use PostgreSQL **17** at data dir
|
|
||||||
`/var/lib/postgresql/17`.
|
|
||||||
|
|
||||||
## Initial setup
|
|
||||||
|
|
||||||
### 1. Provision shared SOPS file (`sus/matrix-cluster.yaml`)
|
|
||||||
|
|
||||||
On a workstation with both yukkop and yukkop-alt age keys available:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo cat /var/lib/matrix-synapse/homeserver.signing.key # on NL (hectic-lab)
|
|
||||||
# Copy the single line value into the buffer for the next step.
|
|
||||||
|
|
||||||
sops sus/matrix-cluster.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
Populate the editor with:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
matrix:
|
|
||||||
signing-key: <paste verbatim signing-key line from NL>
|
|
||||||
postgres-replication-password: <openssl rand -base64 32>
|
|
||||||
object-storage:
|
|
||||||
credentials: |
|
|
||||||
ACCESS_KEY_ID=<verbatim copy from sus/hectic-lab.yaml>
|
|
||||||
SECRET_ACCESS_KEY=<verbatim copy from sus/hectic-lab.yaml>
|
|
||||||
porkbun-api-key: <PORKBUN_API_KEY>
|
|
||||||
porkbun-secret-api-key: <PORKBUN_SECRET_API_KEY>
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify recipients:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sops updatekeys sus/matrix-cluster.yaml
|
|
||||||
sops -d sus/matrix-cluster.yaml | grep -E 'signing-key|porkbun-api-key|object-storage'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: all five keys present, exit 0.
|
|
||||||
|
|
||||||
### 2. Deploy NL primary first
|
|
||||||
|
|
||||||
```sh
|
|
||||||
nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux' --target-host root@128.140.75.58
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify on NL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo systemctl status matrix-synapse postgresql matrix-cluster-replication-password
|
|
||||||
sudo -u postgres psql -c "select rolname, rolreplication from pg_roles where rolname='replication';"
|
|
||||||
# Expected: replication | t
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Seed PL replica with `pg_basebackup`
|
|
||||||
|
|
||||||
On PL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo systemctl stop postgresql
|
|
||||||
sudo rm -rf /var/lib/postgresql/17
|
|
||||||
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
|
|
||||||
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
|
|
||||||
pg_basebackup \
|
|
||||||
-h 128.140.75.58 \
|
|
||||||
-p 5432 \
|
|
||||||
-U replication \
|
|
||||||
-D /var/lib/postgresql/17 \
|
|
||||||
-Fp -Xs -P -R \
|
|
||||||
--no-password
|
|
||||||
```
|
|
||||||
|
|
||||||
`-R` writes `standby.signal` and an initial `primary_conninfo`. The
|
|
||||||
matrix-cluster module's `matrix-cluster-standby-bootstrap` service will
|
|
||||||
overwrite `primary_conninfo` to use a libpq passfile on next boot.
|
|
||||||
|
|
||||||
### 4. Deploy PL standby
|
|
||||||
|
|
||||||
```sh
|
|
||||||
nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux' --target-host root@91.198.166.181
|
|
||||||
sudo systemctl start postgresql
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify streaming on NL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo -u postgres psql -c 'select client_addr, state, sync_state from pg_stat_replication;'
|
|
||||||
# Expected: 91.198.166.181 | streaming | async
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify standby on PL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo -u postgres psql -c 'select pg_is_in_recovery();'
|
|
||||||
# Expected: t
|
|
||||||
sudo systemctl is-active matrix-synapse
|
|
||||||
# Expected: inactive (standby keeps Synapse off)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Remove duplicate S3 credentials from `sus/hectic-lab.yaml`
|
|
||||||
|
|
||||||
Only AFTER NL is confirmed healthy reading from the new shared file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sops sus/hectic-lab.yaml
|
|
||||||
# Delete the matrix/object-storage/credentials block.
|
|
||||||
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
|
|
||||||
```
|
|
||||||
|
|
||||||
## Normal operations
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# NL: replication health
|
|
||||||
sudo -u postgres psql -c 'select * from pg_stat_replication;'
|
|
||||||
# Expected: 1 row, state=streaming, sync_state=async
|
|
||||||
|
|
||||||
# PL: replay status
|
|
||||||
sudo -u postgres psql -c 'select now() - pg_last_xact_replay_timestamp() as lag;'
|
|
||||||
|
|
||||||
# Both: cert renewal
|
|
||||||
sudo systemctl status acme-accord.tube.timer
|
|
||||||
sudo journalctl -u acme-accord.tube.service --since '24 hours ago'
|
|
||||||
|
|
||||||
# Synapse health (NL primary)
|
|
||||||
curl -sf https://accord.tube/_matrix/client/versions | head
|
|
||||||
```
|
|
||||||
|
|
||||||
## Planned failover (NL -> PL)
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# 1. Drain NL: stop accepting writes.
|
|
||||||
sudo systemctl stop matrix-synapse
|
|
||||||
sudo systemctl stop postgresql # ensure no new WAL after this point
|
|
||||||
|
|
||||||
# 2. Promote PL replica.
|
|
||||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
|
|
||||||
# Wait until pg_is_in_recovery() returns f:
|
|
||||||
sudo -u postgres psql -c 'select pg_is_in_recovery();'
|
|
||||||
|
|
||||||
# 3. Make the role switch declarative before rebuilding.
|
|
||||||
# Edit the flake so rebuilds match the promoted database state:
|
|
||||||
# - nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
|
|
||||||
# hectic.generic.matrix-cluster.role = "primary";
|
|
||||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = true;
|
|
||||||
# hectic.generic.matrix-cluster.secretsFile = config.sops.secrets."matrix/secrets".path;
|
|
||||||
# - nixos/system/hectic-lab/hectic-lab.nix:
|
|
||||||
# hectic.generic.matrix-cluster.role = "standby";
|
|
||||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = false;
|
|
||||||
# hectic.generic.matrix-cluster.replication.peerHost = "91.198.166.181";
|
|
||||||
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "128.140.75.58/32" ];
|
|
||||||
# (You will also need a matrix/secrets entry on PL - copy from NL via SOPS.)
|
|
||||||
sudo nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux'
|
|
||||||
sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
|
|
||||||
sudo systemctl status matrix-synapse
|
|
||||||
|
|
||||||
# 4. Swap DNS A record at Porkbun:
|
|
||||||
# accord.tube A 91.198.166.181 (was 128.140.75.58)
|
|
||||||
# TTL: set to 300 in advance of any planned failover.
|
|
||||||
# Porkbun UI: https://porkbun.com/account/domainsSpeedy -> accord.tube -> DNS -> edit A record.
|
|
||||||
# Or via API:
|
|
||||||
sudo curl -sX POST https://api.porkbun.com/api/json/v3/dns/editByNameType/accord.tube/A \
|
|
||||||
-H 'content-type: application/json' \
|
|
||||||
-d "$(jq -n --arg k "$PORKBUN_API_KEY" --arg s "$PORKBUN_SECRET_API_KEY" \
|
|
||||||
'{secretapikey:$s,apikey:$k,content:"91.198.166.181",ttl:"300"}')"
|
|
||||||
|
|
||||||
# 5. Federation smoke test.
|
|
||||||
curl -s 'https://federationtester.matrix.org/api/report?server_name=accord.tube' | jq .FederationOK
|
|
||||||
# Expected: true
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected after the rebuilds:
|
|
||||||
|
|
||||||
- `bfs.poland.xray` evaluates and runs as `role = "primary"`.
|
|
||||||
- `hectic-lab` evaluates as `role = "standby"` with Synapse forced off.
|
|
||||||
- Future `nixos-rebuild` runs preserve the promoted topology instead of reapplying standby settings to PL.
|
|
||||||
|
|
||||||
## Failback (PL -> NL)
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# 1. Stop NL postgres if still up; clear its data dir.
|
|
||||||
sudo systemctl stop postgresql matrix-synapse
|
|
||||||
sudo rm -rf /var/lib/postgresql/17
|
|
||||||
|
|
||||||
# 2. Re-seed NL from PL (now the live primary).
|
|
||||||
sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
|
|
||||||
sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
|
|
||||||
pg_basebackup -h 91.198.166.181 -p 5432 -U replication \
|
|
||||||
-D /var/lib/postgresql/17 -Fp -Xs -P -R --no-password
|
|
||||||
|
|
||||||
# 3. Temporarily flip roles in the flake:
|
|
||||||
# - hectic-lab.nix: role = "standby"; peerHost = "91.198.166.181";
|
|
||||||
# - bfs.poland.xray.nix: role = "primary"; peerHost = "128.140.75.58";
|
|
||||||
# Rebuild both.
|
|
||||||
|
|
||||||
# 4. Once NL is streaming green, do the reverse failover dance:
|
|
||||||
sudo systemctl stop matrix-synapse # on PL
|
|
||||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote # on NL
|
|
||||||
# Then revert the flake role assignments back to NL=primary / PL=standby and
|
|
||||||
# rebuild both hosts.
|
|
||||||
|
|
||||||
# 5. Swap DNS back at Porkbun (A -> 128.140.75.58).
|
|
||||||
```
|
|
||||||
|
|
||||||
## Disaster recovery (NL permanently lost)
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# 1. Promote PL as the new permanent primary.
|
|
||||||
sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
|
|
||||||
|
|
||||||
# 2. Edit nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
|
|
||||||
# hectic.generic.matrix-cluster.role = "primary";
|
|
||||||
# hectic.generic.matrix-cluster.overrideEnableSynapse = lib.mkForce null;
|
|
||||||
# hectic.generic.matrix-cluster.replication.peerHost = "<new-standby-ip>";
|
|
||||||
# hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "<new-standby-ip>/32" ];
|
|
||||||
|
|
||||||
# 3. Provision a new host (any region with Porkbun-managed DNS) and import
|
|
||||||
# self.nixosModules.matrix-cluster with role = "standby" pointed at PL's IP.
|
|
||||||
|
|
||||||
# 4. Bootstrap the new standby via pg_basebackup from PL exactly as in
|
|
||||||
# "Initial setup" step 3, replacing 128.140.75.58 with PL's IP.
|
|
||||||
|
|
||||||
# 5. Update Porkbun A record to PL's IP permanently.
|
|
||||||
```
|
|
||||||
@@ -9,16 +9,11 @@
|
|||||||
config,
|
config,
|
||||||
...
|
...
|
||||||
}: let
|
}: let
|
||||||
matrixBackend = "https://128.140.75.58";
|
|
||||||
matrixHost = "accord.tube";
|
matrixHost = "accord.tube";
|
||||||
jitsiHost = "meet.accord.tube";
|
jitsiHost = "meet.accord.tube";
|
||||||
elementEntryDomain = "element.bfs.band";
|
elementEntryDomain = "element.bfs.band";
|
||||||
polandEntryDomain = "bfs.band";
|
polandEntryDomain = "bfs.band";
|
||||||
backendProxyConfig = ''
|
matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
|
||||||
proxy_ssl_server_name on;
|
|
||||||
proxy_ssl_name ${matrixHost};
|
|
||||||
proxy_set_header Host ${matrixHost};
|
|
||||||
'';
|
|
||||||
in {
|
in {
|
||||||
imports = [
|
imports = [
|
||||||
self.nixosModules.xray-system
|
self.nixosModules.xray-system
|
||||||
@@ -46,7 +41,6 @@ in {
|
|||||||
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
|
credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
|
||||||
};
|
};
|
||||||
replication = {
|
replication = {
|
||||||
peerHost = "128.140.75.58";
|
|
||||||
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
|
passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
|
||||||
};
|
};
|
||||||
acme = {
|
acme = {
|
||||||
@@ -71,11 +65,31 @@ in {
|
|||||||
hostName = jitsiHost;
|
hostName = jitsiHost;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# NOTE(yukkop): disk was provisioned outside disko, so the expected partition
|
||||||
|
# label does not exist. Pin root to the live filesystem UUID so stage 1 can
|
||||||
|
# mount `/` reliably.
|
||||||
|
fileSystems."/" = lib.mkForce {
|
||||||
|
device = "/dev/disk/by-uuid/06b48ef1-a1eb-428d-821c-90c96a624542";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
security.acme = {
|
security.acme = {
|
||||||
acceptTerms = true;
|
acceptTerms = true;
|
||||||
defaults.email = "security@bfs.band";
|
defaults.email = "security@bfs.band";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# NOTE(yukkop): this host gets an IPv6 route via RA, but object storage
|
||||||
|
# fetches to hel1.your-objectstorage.com currently stall over IPv6 while
|
||||||
|
# IPv4 works. Synapse's S3 media backend uses getaddrinfo ordering, so
|
||||||
|
# prefer IPv4 here to keep Element media downloads responsive.
|
||||||
|
environment.etc."gai.conf".text = ''
|
||||||
|
precedence ::ffff:0:0/96 100
|
||||||
|
'';
|
||||||
|
|
||||||
|
systemd.services.matrix-synapse.restartTriggers = [
|
||||||
|
config.environment.etc."gai.conf".source
|
||||||
|
];
|
||||||
|
|
||||||
services.nginx = {
|
services.nginx = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
|
||||||
@@ -112,19 +126,17 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
locations."= /livekit/jwt" = {
|
locations."= /livekit/jwt" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/jwt";
|
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
|
||||||
extraConfig = backendProxyConfig;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /livekit/jwt/" = {
|
locations."^~ /livekit/jwt/" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/jwt/";
|
proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
|
||||||
extraConfig = backendProxyConfig;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
locations."= /livekit/sfu" = {
|
locations."= /livekit/sfu" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/sfu";
|
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
|
||||||
proxyWebsockets = true;
|
proxyWebsockets = true;
|
||||||
extraConfig = backendProxyConfig + ''
|
extraConfig = ''
|
||||||
proxy_send_timeout 120;
|
proxy_send_timeout 120;
|
||||||
proxy_read_timeout 120;
|
proxy_read_timeout 120;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
@@ -135,9 +147,9 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /livekit/sfu/" = {
|
locations."^~ /livekit/sfu/" = {
|
||||||
proxyPass = "${matrixBackend}/livekit/sfu/";
|
proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
|
||||||
proxyWebsockets = true;
|
proxyWebsockets = true;
|
||||||
extraConfig = backendProxyConfig + ''
|
extraConfig = ''
|
||||||
proxy_send_timeout 120;
|
proxy_send_timeout 120;
|
||||||
proxy_read_timeout 120;
|
proxy_read_timeout 120;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
@@ -148,13 +160,14 @@ in {
|
|||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /_matrix/" = {
|
locations."^~ /_matrix/" = {
|
||||||
proxyPass = "${matrixBackend}/_matrix/";
|
proxyPass = "http://127.0.0.1:8008";
|
||||||
extraConfig = backendProxyConfig;
|
extraConfig = ''
|
||||||
|
client_max_body_size ${config.hectic.generic.matrix-cluster.maxUploadSize};
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
locations."^~ /_synapse/client/" = {
|
locations."^~ /_synapse/client/" = {
|
||||||
proxyPass = "${matrixBackend}/_synapse/client/";
|
proxyPass = "http://127.0.0.1:8008";
|
||||||
extraConfig = backendProxyConfig;
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -164,7 +177,7 @@ in {
|
|||||||
|
|
||||||
locations."= /config.${elementEntryDomain}.json".return = "302 /config.json";
|
locations."= /config.${elementEntryDomain}.json".return = "302 /config.json";
|
||||||
|
|
||||||
root = pkgs.element-web.override {
|
root = pkgs.hectic.element-web.override {
|
||||||
conf = {
|
conf = {
|
||||||
default_server_config = {
|
default_server_config = {
|
||||||
"m.homeserver".base_url = "https://${polandEntryDomain}";
|
"m.homeserver".base_url = "https://${polandEntryDomain}";
|
||||||
@@ -176,6 +189,8 @@ in {
|
|||||||
preferred_domain = jitsiHost;
|
preferred_domain = jitsiHost;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
hectic.videoMessages.enabled = true;
|
||||||
|
|
||||||
room_directory.servers = [ matrixHost ];
|
room_directory.servers = [ matrixHost ];
|
||||||
|
|
||||||
default_theme = "dark";
|
default_theme = "dark";
|
||||||
@@ -189,41 +204,41 @@ in {
|
|||||||
key = "matrix/signing-key";
|
key = "matrix/signing-key";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/postgres-replication-password" = {
|
sops.secrets."matrix/postgres-replication-password" = {
|
||||||
key = "matrix/postgres-replication-password";
|
key = "matrix/postgres-replication-password";
|
||||||
owner = "postgres";
|
owner = "postgres";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/object-storage/credentials" = {
|
sops.secrets."matrix/object-storage/credentials" = {
|
||||||
key = "matrix/object-storage/credentials";
|
key = "matrix/object-storage/credentials";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/secrets" = {
|
sops.secrets."matrix/secrets" = {
|
||||||
key = "matrix/secrets";
|
key = "matrix/secrets";
|
||||||
owner = "matrix-synapse";
|
owner = "matrix-synapse";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/turn-secret" = {
|
sops.secrets."matrix/turn-secret" = {
|
||||||
key = "matrix/turn-secret";
|
key = "matrix/turn-secret";
|
||||||
owner = "turnserver";
|
owner = "turnserver";
|
||||||
group = "turnserver";
|
group = "turnserver";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/porkbun-api-key" = {
|
sops.secrets."matrix/porkbun-api-key" = {
|
||||||
key = "matrix/porkbun-api-key";
|
key = "matrix/porkbun-api-key";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
sops.secrets."matrix/porkbun-secret-api-key" = {
|
sops.secrets."matrix/porkbun-secret-api-key" = {
|
||||||
key = "matrix/porkbun-secret-api-key";
|
key = "matrix/porkbun-secret-api-key";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
sopsFile = "${flake}/sus/matrix-cluster.yaml";
|
sopsFile = matrixClusterSopsFile;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
domain,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
hectic.services.attic = {
|
||||||
|
enable = true;
|
||||||
|
hostName = "cache.${domain}";
|
||||||
|
port = 8081;
|
||||||
|
environmentFile = config.sops.secrets."atticd/environment".path;
|
||||||
|
storage = {
|
||||||
|
bucket = "cache-hectic-lab";
|
||||||
|
endpoint = "https://hel1.your-objectstorage.com";
|
||||||
|
region = "hel1";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
services.nginx.virtualHosts."cache.${domain}" = {
|
||||||
|
enableACME = true;
|
||||||
|
forceSSL = true;
|
||||||
|
extraConfig = ''
|
||||||
|
client_max_body_size 0;
|
||||||
|
'';
|
||||||
|
locations."/" = {
|
||||||
|
proxyPass = "http://127.0.0.1:8081";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
domain,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
enteDomain = "ente.${domain}";
|
||||||
|
in {
|
||||||
|
hectic.services.ente = {
|
||||||
|
enable = true;
|
||||||
|
apiDomain = "api.${enteDomain}";
|
||||||
|
disableRegistration = false;
|
||||||
|
|
||||||
|
domains = {
|
||||||
|
accounts = "accounts.${enteDomain}";
|
||||||
|
cast = "cast.${enteDomain}";
|
||||||
|
albums = "albums.${enteDomain}";
|
||||||
|
photos = "photos.${enteDomain}";
|
||||||
|
};
|
||||||
|
|
||||||
|
smtp = {
|
||||||
|
enable = true;
|
||||||
|
host = "mail.${domain}";
|
||||||
|
email = "security@${domain}";
|
||||||
|
};
|
||||||
|
|
||||||
|
storage = {
|
||||||
|
bucket = "ente-hectic-lab";
|
||||||
|
endpoint = "https://hel1.your-objectstorage.com";
|
||||||
|
region = "hel1";
|
||||||
|
};
|
||||||
|
|
||||||
|
secrets = {
|
||||||
|
encryptionKeyFile = config.sops.secrets."ente/key-encryption".path;
|
||||||
|
hashKeyFile = config.sops.secrets."ente/key-hash".path;
|
||||||
|
jwtSecretFile = config.sops.secrets."ente/jwt-secret".path;
|
||||||
|
s3AccessKeyFile = config.sops.secrets."ente/s3-access-key".path;
|
||||||
|
s3SecretKeyFile = config.sops.secrets."ente/s3-secret-key".path;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
{ pkgs, ... }:
|
||||||
|
let
|
||||||
|
port = 22222;
|
||||||
|
stateDir = "/var/lib/experimental-sshd";
|
||||||
|
configFile = pkgs.writeText "experimental-sshd_config" ''
|
||||||
|
Port 22222
|
||||||
|
ListenAddress 0.0.0.0
|
||||||
|
ListenAddress ::
|
||||||
|
|
||||||
|
HostKey ${stateDir}/ssh_host_ed25519_key
|
||||||
|
HostKey ${stateDir}/ssh_host_rsa_key
|
||||||
|
|
||||||
|
PasswordAuthentication no
|
||||||
|
KbdInteractiveAuthentication no
|
||||||
|
PubkeyAuthentication yes
|
||||||
|
PermitRootLogin prohibit-password
|
||||||
|
UsePAM yes
|
||||||
|
AuthenticationMethods publickey
|
||||||
|
AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u
|
||||||
|
LogLevel DEBUG3
|
||||||
|
|
||||||
|
VersionAddendum none
|
||||||
|
HostKeyAlgorithms rsa-sha2-512,rsa-sha2-256,ssh-ed25519
|
||||||
|
KexAlgorithms curve25519-sha256,diffie-hellman-group14-sha256
|
||||||
|
Ciphers aes256-ctr,aes128-ctr
|
||||||
|
MACs hmac-sha2-512,hmac-sha2-256
|
||||||
|
'';
|
||||||
|
|
||||||
|
keygenScript = pkgs.writeShellScript "experimental-sshd-keygen" ''
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
${pkgs.coreutils}/bin/mkdir -p -m 0700 ${stateDir}
|
||||||
|
|
||||||
|
if [ ! -f ${stateDir}/ssh_host_ed25519_key ]; then
|
||||||
|
${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f ${stateDir}/ssh_host_ed25519_key -N ""
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f ${stateDir}/ssh_host_rsa_key ]; then
|
||||||
|
${pkgs.openssh}/bin/ssh-keygen -t rsa -b 4096 -f ${stateDir}/ssh_host_rsa_key -N ""
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
environment.etc."ssh/experimental-sshd_config".source = configFile;
|
||||||
|
|
||||||
|
networking.firewall.allowedTCPPorts = [ port ];
|
||||||
|
|
||||||
|
systemd.services.experimental-sshd-keygen = {
|
||||||
|
description = "Generate experimental SSH host keys";
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
StateDirectory = "experimental-sshd";
|
||||||
|
ExecStart = keygenScript;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services.experimental-sshd = {
|
||||||
|
description = "Experimental SSH daemon";
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
wants = [ "experimental-sshd-keygen.service" ];
|
||||||
|
after = [ "network.target" "experimental-sshd-keygen.service" ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "simple";
|
||||||
|
StateDirectory = "experimental-sshd";
|
||||||
|
RuntimeDirectory = "experimental-sshd";
|
||||||
|
ExecStart = "${pkgs.openssh}/bin/sshd -D -e -f /etc/ssh/experimental-sshd_config";
|
||||||
|
};
|
||||||
|
preStart = ''
|
||||||
|
${pkgs.openssh}/bin/sshd -t -f /etc/ssh/experimental-sshd_config
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -15,7 +15,7 @@ with builtins;
|
|||||||
with lib;
|
with lib;
|
||||||
let
|
let
|
||||||
domain = "hectic-lab.com";
|
domain = "hectic-lab.com";
|
||||||
matrixDomain = "accord.tube";
|
sshPort = 22;
|
||||||
mailUserNames = [
|
mailUserNames = [
|
||||||
"security"
|
"security"
|
||||||
"founders"
|
"founders"
|
||||||
@@ -25,6 +25,7 @@ let
|
|||||||
"iana-perlyk"
|
"iana-perlyk"
|
||||||
"snuff"
|
"snuff"
|
||||||
"antoshka"
|
"antoshka"
|
||||||
|
"evgenii-kazakov"
|
||||||
];
|
];
|
||||||
mkMailPasswordSecret = name: {
|
mkMailPasswordSecret = name: {
|
||||||
name = "mailserver/${name}/hashedPassword";
|
name = "mailserver/${name}/hashedPassword";
|
||||||
@@ -36,9 +37,12 @@ let
|
|||||||
hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path;
|
hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
sslOpts = {
|
mkEnteSecret = name: {
|
||||||
sslCertificate = config.sops.secrets."ssl/porkbun/${domain}/domain.cert.pem".path;
|
name = "ente/${name}";
|
||||||
sslCertificateKey = config.sops.secrets."ssl/porkbun/${domain}/private.key.pem".path;
|
value = {
|
||||||
|
owner = "ente";
|
||||||
|
group = "ente";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
in {
|
in {
|
||||||
imports = [
|
imports = [
|
||||||
@@ -51,9 +55,12 @@ in {
|
|||||||
|
|
||||||
inputs.hectic-landing.nixosModules.hectic-landing
|
inputs.hectic-landing.nixosModules.hectic-landing
|
||||||
|
|
||||||
|
(import ./attic.nix { inherit flake self inputs domain; })
|
||||||
(import ./containers.nix { inherit flake self inputs; })
|
(import ./containers.nix { inherit flake self inputs; })
|
||||||
(import ./mechabellum.nix { inherit flake self inputs domain sslOpts; })
|
./experimental-sshd.nix
|
||||||
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain sslOpts; })
|
(import ./ente.nix { inherit domain; })
|
||||||
|
(import ./mechabellum.nix { inherit flake self inputs domain; })
|
||||||
|
(import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
|
||||||
];
|
];
|
||||||
|
|
||||||
services.hectic-landing = {
|
services.hectic-landing = {
|
||||||
@@ -74,6 +81,7 @@ in {
|
|||||||
enable = true;
|
enable = true;
|
||||||
networkMatchConfigName = "enp1s0";
|
networkMatchConfigName = "enp1s0";
|
||||||
ipv4 = "128.140.75.58";
|
ipv4 = "128.140.75.58";
|
||||||
|
floatingIpv4 = "78.47.243.0";
|
||||||
ipv6 = "2a01:4f8:c2c:d54a";
|
ipv6 = "2a01:4f8:c2c:d54a";
|
||||||
};
|
};
|
||||||
services.matrix = {
|
services.matrix = {
|
||||||
@@ -100,6 +108,7 @@ in {
|
|||||||
'';
|
'';
|
||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
|
tcpdump
|
||||||
git
|
git
|
||||||
rsync
|
rsync
|
||||||
python311
|
python311
|
||||||
@@ -110,28 +119,25 @@ in {
|
|||||||
sops = {
|
sops = {
|
||||||
gnupg.sshKeyPaths = [ ];
|
gnupg.sshKeyPaths = [ ];
|
||||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||||
defaultSopsFile = "${flake}/sus/hectic-lab.yaml";
|
defaultSopsFile = flake + "/sus/hectic-lab.yaml";
|
||||||
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
|
secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
|
||||||
"init-postgresql" = {
|
"init-postgresql" = {
|
||||||
key = "init-postgresql";
|
key = "init-postgresql";
|
||||||
};
|
};
|
||||||
"ssl/porkbun/${domain}/domain.cert.pem" = {
|
"atticd/environment" = {};
|
||||||
group = "nginx";
|
|
||||||
mode = "0440";
|
|
||||||
};
|
|
||||||
"ssl/porkbun/${domain}/private.key.pem" = {
|
|
||||||
group = "nginx";
|
|
||||||
mode = "0440";
|
|
||||||
};
|
|
||||||
"ssl/porkbun/${domain}/public.key.pem" = {
|
|
||||||
group = "nginx";
|
|
||||||
mode = "0440";
|
|
||||||
};
|
|
||||||
"wg-bfs/private-key" = {};
|
"wg-bfs/private-key" = {};
|
||||||
};
|
} // builtins.listToAttrs (map mkEnteSecret [
|
||||||
|
"key-encryption"
|
||||||
|
"key-hash"
|
||||||
|
"jwt-secret"
|
||||||
|
"s3-access-key"
|
||||||
|
"s3-secret-key"
|
||||||
|
]);
|
||||||
};
|
};
|
||||||
|
|
||||||
users.users.root.openssh.authorizedKeys.keys = [
|
users.users.root.openssh.authorizedKeys.keys = [
|
||||||
|
# neuro machine
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
|
||||||
# yukkop
|
# yukkop
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
|
||||||
@@ -158,6 +164,8 @@ in {
|
|||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
services.openssh.ports = [ sshPort ];
|
||||||
|
|
||||||
services.mailserver = {
|
services.mailserver = {
|
||||||
enable = true;
|
enable = true;
|
||||||
domain = domain;
|
domain = domain;
|
||||||
@@ -178,10 +186,10 @@ in {
|
|||||||
|
|
||||||
networking.firewall = {
|
networking.firewall = {
|
||||||
allowedTCPPorts = [
|
allowedTCPPorts = [
|
||||||
|
sshPort # ssh
|
||||||
80
|
80
|
||||||
443
|
443
|
||||||
3306 # mysql
|
3306 # mysql
|
||||||
11012 # gitea ssh
|
|
||||||
25565
|
25565
|
||||||
55228 # ss-bfs
|
55228 # ss-bfs
|
||||||
];
|
];
|
||||||
@@ -204,8 +212,8 @@ in {
|
|||||||
services.nginx = {
|
services.nginx = {
|
||||||
enable = true;
|
enable = true;
|
||||||
# NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module
|
# NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module
|
||||||
# (ACME-managed). See services.hectic-landing above.
|
virtualHosts."store.${domain}" = {
|
||||||
virtualHosts."store.${domain}" = sslOpts // {
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
root = "/var/www/store";
|
root = "/var/www/store";
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
@@ -214,7 +222,8 @@ in {
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
virtualHosts."snuff.${domain}" = sslOpts // {
|
virtualHosts."snuff.${domain}" = {
|
||||||
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
@@ -223,7 +232,8 @@ in {
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
virtualHosts."nrv.${domain}" = sslOpts // {
|
virtualHosts."nrv.${domain}" = {
|
||||||
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
@@ -232,7 +242,8 @@ in {
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
virtualHosts."yukkop.${domain}" = sslOpts // {
|
virtualHosts."yukkop.${domain}" = {
|
||||||
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
@@ -257,10 +268,10 @@ in {
|
|||||||
gitea = {
|
gitea = {
|
||||||
enable = true;
|
enable = true;
|
||||||
package = pkgs.hectic.gitea-heatmap;
|
package = pkgs.hectic.gitea-heatmap;
|
||||||
settings.service.DISABLE_REGISTRATION = false;
|
settings.service.DISABLE_REGISTRATION = true;
|
||||||
settings.server = {
|
settings.server = {
|
||||||
HTTP_PORT = 11011;
|
HTTP_PORT = 11011;
|
||||||
#SSH_PORT = 22;
|
SSH_PORT = sshPort;
|
||||||
SSH_DOMAIN = "hectic-lab.com";
|
SSH_DOMAIN = "hectic-lab.com";
|
||||||
};
|
};
|
||||||
database = {
|
database = {
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
{
|
{
|
||||||
inputs,
|
inputs,
|
||||||
domain,
|
domain,
|
||||||
sslOpts,
|
|
||||||
...
|
...
|
||||||
}: {
|
}: {
|
||||||
pkgs,
|
pkgs,
|
||||||
@@ -29,7 +28,8 @@ in {
|
|||||||
enable = true;
|
enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
services.nginx.virtualHosts."${mechDomain}" = sslOpts // {
|
services.nginx.virtualHosts."${mechDomain}" = {
|
||||||
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
root = inputs.mechabellum-replay-analysis.packages.${system}.frontend;
|
root = inputs.mechabellum-replay-analysis.packages.${system}.frontend;
|
||||||
|
|
||||||
@@ -37,6 +37,7 @@ in {
|
|||||||
proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
|
proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
|
client_max_body_size 500M;
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
flake,
|
flake,
|
||||||
self,
|
self,
|
||||||
domain,
|
domain,
|
||||||
sslOpts,
|
|
||||||
...
|
...
|
||||||
}: { ... }: {
|
}: { ... }: {
|
||||||
hectic.services."sentinèlla" = {
|
hectic.services."sentinèlla" = {
|
||||||
@@ -17,7 +16,8 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
services.nginx = {
|
services.nginx = {
|
||||||
virtualHosts."probe.${domain}" = sslOpts // {
|
virtualHosts."probe.${domain}" = {
|
||||||
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
proxyPass = "http://127.0.0.1:5988";
|
proxyPass = "http://127.0.0.1:5988";
|
||||||
|
|||||||
@@ -15,32 +15,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
|
|||||||
self.overlays.default
|
self.overlays.default
|
||||||
inputs.nix-minecraft.overlay
|
inputs.nix-minecraft.overlay
|
||||||
];
|
];
|
||||||
config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
config.allowUnfreePredicate = pkg:
|
||||||
|
self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
|
||||||
"minecraft-server"
|
"minecraft-server"
|
||||||
"neoforge"
|
"neoforge"
|
||||||
|
|
||||||
"nvidia-x11"
|
"nvidia-x11"
|
||||||
|
|
||||||
"cuda_nvcc"
|
|
||||||
"cuda_cudart"
|
|
||||||
"cuda_cuobjdump"
|
|
||||||
"cuda_cupti"
|
|
||||||
"cuda_nvdisasm"
|
|
||||||
"cuda_cccl"
|
|
||||||
"cuda_nvml_dev"
|
|
||||||
"cuda_nvrtc"
|
|
||||||
"cuda_nvtx"
|
|
||||||
"cuda_profiler_api"
|
|
||||||
|
|
||||||
"libcusparse_lt"
|
|
||||||
"libcublas"
|
|
||||||
"libcufft"
|
|
||||||
"libcufile"
|
|
||||||
"libcurand"
|
|
||||||
"libcusolver"
|
|
||||||
"libnvjitlink"
|
|
||||||
"libcusparse"
|
|
||||||
"cudnn"
|
|
||||||
];
|
];
|
||||||
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
|
# jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
|
||||||
config.permittedInsecurePackages = [
|
config.permittedInsecurePackages = [
|
||||||
|
|||||||
@@ -17,6 +17,11 @@
|
|||||||
|
|
||||||
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
|
ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
|
||||||
|
|
||||||
|
stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
|
||||||
|
pkgs.stdenv.cc.cc.lib
|
||||||
|
pkgs.zlib
|
||||||
|
];
|
||||||
|
|
||||||
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
|
ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
|
||||||
pname = "ollama";
|
pname = "ollama";
|
||||||
version = "0.22.1";
|
version = "0.22.1";
|
||||||
@@ -176,6 +181,19 @@ in {
|
|||||||
openFirewall = false;
|
openFirewall = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
hectic.services.stable-video-diffusion = {
|
||||||
|
enable = true;
|
||||||
|
host = "127.0.0.1";
|
||||||
|
port = 7861;
|
||||||
|
package = pkgs.hectic.stable-video-diffusion-api;
|
||||||
|
device = "cuda";
|
||||||
|
libraryPath = [
|
||||||
|
stableVideoDiffusionLibraryPath
|
||||||
|
"/run/opengl-driver/lib"
|
||||||
|
];
|
||||||
|
openFirewall = false;
|
||||||
|
};
|
||||||
|
|
||||||
networking = {
|
networking = {
|
||||||
networkmanager.enable = true;
|
networkmanager.enable = true;
|
||||||
useDHCP = lib.mkDefault true;
|
useDHCP = lib.mkDefault true;
|
||||||
@@ -263,6 +281,7 @@ in {
|
|||||||
in [
|
in [
|
||||||
#python-ai
|
#python-ai
|
||||||
git
|
git
|
||||||
|
hectic.hiddify-core
|
||||||
neovim
|
neovim
|
||||||
wget
|
wget
|
||||||
ethtool
|
ethtool
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
flake,
|
||||||
|
self,
|
||||||
|
inputs,
|
||||||
|
system ? "x86_64-linux",
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
# Use folder name as name of this system
|
||||||
|
name = builtins.baseNameOf ./.;
|
||||||
|
|
||||||
|
in self.lib.nixpkgs-lib.nixosSystem {
|
||||||
|
pkgs = import inputs.nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
overlays = [ self.overlays.default ];
|
||||||
|
};
|
||||||
|
modules = [
|
||||||
|
{ networking.hostName = name; }
|
||||||
|
(import ./${name}.nix { inherit flake self inputs; })
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
{
|
||||||
|
inputs,
|
||||||
|
self,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
pkgs,
|
||||||
|
modulesPath,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
imports = [
|
||||||
|
(modulesPath + "/profiles/qemu-guest.nix")
|
||||||
|
inputs.sops-nix.nixosModules.sops
|
||||||
|
self.nixosModules.hectic
|
||||||
|
];
|
||||||
|
|
||||||
|
hectic = {
|
||||||
|
archetype.dev.enable = true;
|
||||||
|
hardware.hetzner-cloud = {
|
||||||
|
enable = true;
|
||||||
|
device = "/dev/sda";
|
||||||
|
networkMatchConfigMac = "92:00:08:4a:b0:32";
|
||||||
|
ipv4 = "46.225.237.218";
|
||||||
|
ipv6 = "2a01:4f8:c2c:3b14";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
users.users.root.openssh.authorizedKeys.keys = [
|
||||||
|
''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
|
||||||
|
];
|
||||||
|
|
||||||
|
environment.systemPackages = with pkgs; [
|
||||||
|
git
|
||||||
|
rsync
|
||||||
|
];
|
||||||
|
}
|
||||||
+37
-11
@@ -1,18 +1,21 @@
|
|||||||
# db-tool
|
# db-tool
|
||||||
|
|
||||||
PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries.
|
PostgreSQL utility package exposing separate development and operations binaries.
|
||||||
|
`db-dev` preserves the current development workflow via the `database` binary;
|
||||||
|
`db-ops` provides target-safe operational helpers such as secrets hydration.
|
||||||
|
|
||||||
## Provided Binaries
|
## Provided Binaries
|
||||||
|
|
||||||
| Binary | Description |
|
| Binary | Description |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `database` | Main script for managing migrations, deployments, and logs. |
|
| `database` | Main `db-dev` script for managing local development databases, migrations, deployments, and logs. |
|
||||||
|
| `db-ops` | Operational helper binary for target/runtime database actions. |
|
||||||
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
|
| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
|
||||||
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
|
| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |
|
||||||
|
|
||||||
## Required Environment Variables
|
## Required Environment Variables
|
||||||
|
|
||||||
These variables must be set for `db-tool` to function.
|
These variables must be set for `db-dev` / `database` to function.
|
||||||
|
|
||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
@@ -39,7 +42,7 @@ These variables must be set for `db-tool` to function.
|
|||||||
|
|
||||||
## Postgres Package Override
|
## Postgres Package Override
|
||||||
|
|
||||||
By default, `db-tool`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
|
By default, `db-dev`/`db-ops`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
let
|
let
|
||||||
@@ -48,7 +51,8 @@ let
|
|||||||
]);
|
]);
|
||||||
in {
|
in {
|
||||||
packages = [
|
packages = [
|
||||||
(pkgs.hectic."db-tool".override { postgresql = myPg; })
|
(pkgs.hectic."db-dev".override { postgresql = myPg; })
|
||||||
|
(pkgs.hectic."db-ops".override { postgresql = myPg; })
|
||||||
(pkgs.hectic."postgres-init".override { postgresql = myPg; })
|
(pkgs.hectic."postgres-init".override { postgresql = myPg; })
|
||||||
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
|
(pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
|
||||||
];
|
];
|
||||||
@@ -66,6 +70,28 @@ The `pull_staging` subcommand allows importing data from a remote staging enviro
|
|||||||
|
|
||||||
If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr.
|
If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr.
|
||||||
|
|
||||||
|
## normalize-backup Contract
|
||||||
|
|
||||||
|
The `normalize-backup` subcommand converts a physical PostgreSQL backup into a
|
||||||
|
local-development restore artifact without modifying the input backup:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
database normalize-backup [OPTIONS] [path]
|
||||||
|
```
|
||||||
|
|
||||||
|
The input `path` defaults to `${LOCAL_DIR}/focus/postgresql-backup` and must be a
|
||||||
|
backup directory compatible with `database restore`, containing `base.tar.gz`
|
||||||
|
and optionally `pg_wal.tar.gz`. The output defaults to a normalized backup path
|
||||||
|
and can be overridden with `--output <path>`. The output directory must not be
|
||||||
|
the same path as the input and must not be inside the input directory.
|
||||||
|
|
||||||
|
Normalization extracts the physical backup into temporary PGDATA, replaces
|
||||||
|
production PostgreSQL access/configuration with local restore-safe
|
||||||
|
`postgresql.conf` and `pg_hba.conf` files, removes standby/recovery/runtime
|
||||||
|
leftovers, starts the cluster locally, ensures the requested `--role` and
|
||||||
|
`--database` exist, stops cleanly, and repacks a backup directory that can be
|
||||||
|
used by `database restore`.
|
||||||
|
|
||||||
## Subcommands
|
## Subcommands
|
||||||
|
|
||||||
- `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success.
|
- `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success.
|
||||||
@@ -73,20 +99,21 @@ If any of these variables are missing when `pull_staging` is invoked, the tool w
|
|||||||
- `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`.
|
- `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`.
|
||||||
- `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster.
|
- `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster.
|
||||||
- `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`.
|
- `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`.
|
||||||
|
- `normalize-backup`: Rewrite a physical backup artifact for local restore use.
|
||||||
- `pull_staging`: Import data from the staging environment based on the env contract.
|
- `pull_staging`: Import data from the staging environment based on the env contract.
|
||||||
- `init`: Wrapper around `postgres-init` to start the cluster.
|
- `init`: Wrapper around `postgres-init` to start the cluster.
|
||||||
- `migrator`: Directly invoke the migration tool with the correct environment context.
|
- `migrator`: Directly invoke the migration tool with the correct environment context.
|
||||||
|
|
||||||
## shellHook Example
|
## shellHook Example
|
||||||
|
|
||||||
To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
|
To use `db-dev` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
{
|
{
|
||||||
# ...
|
# ...
|
||||||
devShells.default = pkgs.mkShell {
|
devShells.default = pkgs.mkShell {
|
||||||
packages = [
|
packages = [
|
||||||
pkgs.hectic.db-tool
|
pkgs.hectic.db-dev
|
||||||
pkgs.hectic.postgres-init
|
pkgs.hectic.postgres-init
|
||||||
pkgs.hectic.postgres-cleanup
|
pkgs.hectic.postgres-cleanup
|
||||||
];
|
];
|
||||||
@@ -111,7 +138,7 @@ To use `db-tool` in a Nix development shell, add the following to your `flake.ni
|
|||||||
|
|
||||||
## hectic Bundle
|
## hectic Bundle
|
||||||
|
|
||||||
`db-tool` and `migrator` apply a single bundle of SQL files that bootstrap the
|
`db-dev`, `db-ops`, and `migrator` apply a single bundle of SQL files that bootstrap the
|
||||||
`hectic` schema. The bundle lives in
|
`hectic` schema. The bundle lives in
|
||||||
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
|
[`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
|
||||||
contract, file layout, and the `self.lib.hectic.*` Nix API.
|
contract, file layout, and the `self.lib.hectic.*` Nix API.
|
||||||
@@ -167,6 +194,7 @@ ALTER DATABASE mydb SET hectic.inheritance_extra_excluded_schemas = 'legacy,etl'
|
|||||||
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
|
| `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
|
||||||
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
|
| `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
|
||||||
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
|
| `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
|
||||||
|
| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. |
|
||||||
|
|
||||||
The bundle is idempotent — repeated application is safe.
|
The bundle is idempotent — repeated application is safe.
|
||||||
|
|
||||||
@@ -187,15 +215,13 @@ directly against the paths exposed by `self.lib.hectic.*.path`:
|
|||||||
|
|
||||||
```nix
|
```nix
|
||||||
services.postgresql.initialScript = pkgs.writeText "hectic-init.sql" ''
|
services.postgresql.initialScript = pkgs.writeText "hectic-init.sql" ''
|
||||||
|
\i ${self.lib.hectic.version.path}
|
||||||
\i ${self.lib.hectic.secret.path}
|
\i ${self.lib.hectic.secret.path}
|
||||||
\i ${self.lib.hectic.migration.path}
|
\i ${self.lib.hectic.migration.path}
|
||||||
\i ${self.lib.hectic.inheritance.path}
|
\i ${self.lib.hectic.inheritance.path}
|
||||||
'';
|
'';
|
||||||
```
|
```
|
||||||
|
|
||||||
The version file (`self.lib.hectic.version`) is templated and only exposes
|
|
||||||
`.sql` (a string). Materialize it with `pkgs.writeText` if a path is needed.
|
|
||||||
|
|
||||||
## Exit Codes
|
## Exit Codes
|
||||||
|
|
||||||
| Code | Meaning |
|
| Code | Meaning |
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
: "${REMAINING_ARGS:=}"
|
: "${REMAINING_ARGS:=}"
|
||||||
|
|
||||||
: "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}"
|
: "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}"
|
||||||
|
: "${DEFAULT_NORMALIZED_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup-normalized}"
|
||||||
|
|
||||||
: "${SCRIPT_NAME:=$(basename "$0")}"
|
: "${SCRIPT_NAME:=$(basename "$0")}"
|
||||||
SCRIPT_NAME=${SCRIPT_NAME%%.sh}
|
SCRIPT_NAME=${SCRIPT_NAME%%.sh}
|
||||||
@@ -186,6 +187,9 @@ ${BGREEN}Database Subcommands:${NC}
|
|||||||
|
|
||||||
${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup
|
${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup
|
||||||
|
|
||||||
|
${BCYAN}normalize-backup${NC} ${CYAN}[OPTIONS] [PATH]$NC
|
||||||
|
Normalize a raw physical backup for local restore/diff
|
||||||
|
|
||||||
${BCYAN}backup${NC} Create database backup
|
${BCYAN}backup${NC} Create database backup
|
||||||
Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC
|
Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC
|
||||||
|
|
||||||
@@ -246,6 +250,7 @@ ${BGREEN}Examples:${NC}
|
|||||||
$SCRIPT_NAME init Initialize PostgreSQL only
|
$SCRIPT_NAME init Initialize PostgreSQL only
|
||||||
$SCRIPT_NAME restore Restore from default backup
|
$SCRIPT_NAME restore Restore from default backup
|
||||||
$SCRIPT_NAME restore /path/to/backup Restore from specific path
|
$SCRIPT_NAME restore /path/to/backup Restore from specific path
|
||||||
|
$SCRIPT_NAME normalize-backup /path/to/raw-backup
|
||||||
$SCRIPT_NAME backup Create backup
|
$SCRIPT_NAME backup Create backup
|
||||||
$SCRIPT_NAME log Show database logs
|
$SCRIPT_NAME log Show database logs
|
||||||
$SCRIPT_NAME log list List available log files
|
$SCRIPT_NAME log list List available log files
|
||||||
@@ -421,6 +426,43 @@ EOF
|
|||||||
)" | "$PAGER_OR_CAT"
|
)" | "$PAGER_OR_CAT"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
help_normalize_backup() {
|
||||||
|
# shellcheck disable=SC2059
|
||||||
|
printf "$(cat <<EOF
|
||||||
|
${BGREEN}Usage:${NC} $SCRIPT_NAME normalize-backup [OPTIONS] [path]
|
||||||
|
|
||||||
|
Normalize a raw PostgreSQL physical backup for local development.
|
||||||
|
|
||||||
|
This command leaves the input backup untouched. It extracts $BBLACK\`base.tar.gz\`$NC
|
||||||
|
and optional $BBLACK\`pg_wal.tar.gz\`$NC into temporary PGDATA, writes local
|
||||||
|
$BBLACK\`postgresql.conf\`$NC and $BBLACK\`pg_hba.conf\`$NC files, removes standby,
|
||||||
|
recovery, and runtime leftovers, starts the cluster locally, ensures a login role
|
||||||
|
and database exist, stops cleanly, then repacks a normalized backup directory.
|
||||||
|
|
||||||
|
${BGREEN}Arguments:${NC}
|
||||||
|
${BCYAN}path$NC Input backup directory (optional)
|
||||||
|
Defaults to $BBLACK$DEFAULT_BACKUP_PATH$NC
|
||||||
|
|
||||||
|
${BGREEN}Options:${NC}
|
||||||
|
$BCYAN-o$NC, $BCYAN--output$NC ${CYAN}<path>$NC Output backup directory
|
||||||
|
Defaults to $BBLACK$DEFAULT_NORMALIZED_BACKUP_PATH$NC
|
||||||
|
$BCYAN--role$NC ${CYAN}<name>$NC Login role to ensure (default $BBLACK\$(id -un)$NC)
|
||||||
|
$BCYAN--database$NC ${CYAN}<name>$NC Database to ensure (default $BBLACK\${PG_DATABASE:-testdb}$NC)
|
||||||
|
$BCYAN--admin-role$NC ${CYAN}<name>$NC Role used for local maintenance
|
||||||
|
Defaults to $BBLACK\${URI_USER:-postgres}$NC
|
||||||
|
$BCYAN-h$NC, $BCYAN--help$NC Show this help message
|
||||||
|
|
||||||
|
${BGREEN}Files Created:${NC}
|
||||||
|
- ${BBLACK}\`base.tar.gz\`$NC: Normalized database cluster backup
|
||||||
|
|
||||||
|
${BGREEN}Examples:${NC}
|
||||||
|
$SCRIPT_NAME normalize-backup /path/to/raw-backup
|
||||||
|
$SCRIPT_NAME normalize-backup --output focus/postgresql-backup-normalized
|
||||||
|
|
||||||
|
EOF
|
||||||
|
)" | "$PAGER_OR_CAT"
|
||||||
|
}
|
||||||
|
|
||||||
help_diff() {
|
help_diff() {
|
||||||
# shellcheck disable=SC2059
|
# shellcheck disable=SC2059
|
||||||
printf "$(cat <<EOF
|
printf "$(cat <<EOF
|
||||||
@@ -441,6 +483,8 @@ ${BGREEN}Arguments:
|
|||||||
${BGREEN}Options:${NC}
|
${BGREEN}Options:${NC}
|
||||||
$BCYAN--tables${NC} ${CYAN}<list>${NC} Comma-separated list of tables to diff
|
$BCYAN--tables${NC} ${CYAN}<list>${NC} Comma-separated list of tables to diff
|
||||||
Example: --tables users,orders,products
|
Example: --tables users,orders,products
|
||||||
|
$BCYAN--ignore-migration-fail${NC}
|
||||||
|
Continue diff even if DB1 migrations fail
|
||||||
$BCYAN-m${NC}, $BCYAN--mock${NC} Mock external API calls when hydrating DB2
|
$BCYAN-m${NC}, $BCYAN--mock${NC} Mock external API calls when hydrating DB2
|
||||||
Replaces HTTP-calling functions with stubs/test data
|
Replaces HTTP-calling functions with stubs/test data
|
||||||
$BCYAN-h${NC}, $BCYAN--help${NC} Show this help message
|
$BCYAN-h${NC}, $BCYAN--help${NC} Show this help message
|
||||||
@@ -464,6 +508,7 @@ ${BGREEN}Examples:${NC}
|
|||||||
$SCRIPT_NAME diff Compare using default backup
|
$SCRIPT_NAME diff Compare using default backup
|
||||||
$SCRIPT_NAME diff /path/to/backup Compare using specific backup
|
$SCRIPT_NAME diff /path/to/backup Compare using specific backup
|
||||||
$SCRIPT_NAME diff --tables users,orders Compare specific tables
|
$SCRIPT_NAME diff --tables users,orders Compare specific tables
|
||||||
|
$SCRIPT_NAME diff --ignore-migration-fail Continue despite DB1 migration failure
|
||||||
$SCRIPT_NAME diff -m Compare with external APIs mocked
|
$SCRIPT_NAME diff -m Compare with external APIs mocked
|
||||||
$SCRIPT_NAME diff log Show logs from diff operation
|
$SCRIPT_NAME diff log Show logs from diff operation
|
||||||
|
|
||||||
@@ -805,6 +850,11 @@ subcommand_restore() {
|
|||||||
RESTORE_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
|
RESTORE_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
local restore_database="${PG_DATABASE:-}"
|
||||||
|
if [ -f "${RESTORE_BACKUP_PATH:?}/database_name" ]; then
|
||||||
|
restore_database=$(tr -d '\n' < "${RESTORE_BACKUP_PATH:?}/database_name")
|
||||||
|
fi
|
||||||
|
|
||||||
postgres-cleanup
|
postgres-cleanup
|
||||||
|
|
||||||
local data="${PG_WORKING_DIR:?}/data"
|
local data="${PG_WORKING_DIR:?}/data"
|
||||||
@@ -817,12 +867,243 @@ subcommand_restore() {
|
|||||||
tar -xzf "${RESTORE_BACKUP_PATH:?}/pg_wal.tar.gz" -C "${data}/pg_wal"
|
tar -xzf "${RESTORE_BACKUP_PATH:?}/pg_wal.tar.gz" -C "${data}/pg_wal"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
env PG_REUSE= postgres-init
|
env PG_REUSE= PG_DATABASE="$restore_database" postgres-init
|
||||||
|
|
||||||
rm -f "${data}/standby.signal" "${data}/recovery.signal"
|
rm -f "${data}/standby.signal" "${data}/recovery.signal"
|
||||||
restore_namespace
|
restore_namespace
|
||||||
}
|
}
|
||||||
|
|
||||||
|
___sql_literal() {
|
||||||
|
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/''/g")"
|
||||||
|
}
|
||||||
|
|
||||||
|
___normalize_backup_remove_leftovers() {
|
||||||
|
local pgdata="$1"
|
||||||
|
rm -f \
|
||||||
|
"$pgdata/postmaster.pid" \
|
||||||
|
"$pgdata/postmaster.opts" \
|
||||||
|
"$pgdata/recovery.signal" \
|
||||||
|
"$pgdata/standby.signal" \
|
||||||
|
"$pgdata/backup_label.old"
|
||||||
|
rm -f "$pgdata/pg_wal/archive_status"/*.ready 2>/dev/null || :
|
||||||
|
}
|
||||||
|
|
||||||
|
___normalize_backup_cleanup() {
|
||||||
|
local tmpdir="$1"
|
||||||
|
local pgdata="$2"
|
||||||
|
if [ -n "$pgdata" ] && [ -d "$pgdata" ]; then
|
||||||
|
pg_ctl -D "$pgdata" -m fast -w stop >/dev/null 2>&1 || :
|
||||||
|
fi
|
||||||
|
if [ -n "$tmpdir" ]; then
|
||||||
|
rm -rf "$tmpdir"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
___normalize_backup_refresh_collation() {
|
||||||
|
local sockdir="$1"
|
||||||
|
local port="$2"
|
||||||
|
local admin_role="$3"
|
||||||
|
local database_name="$4"
|
||||||
|
|
||||||
|
psql -h "$sockdir" -p "$port" -U "$admin_role" -d postgres \
|
||||||
|
-v ON_ERROR_STOP=1 -c "ALTER DATABASE \"$database_name\" REFRESH COLLATION VERSION;" \
|
||||||
|
>/dev/null 2>&1 || :
|
||||||
|
}
|
||||||
|
|
||||||
|
subcommand_normalize_backup() {
|
||||||
|
change_namespace 'db normalize-backup'
|
||||||
|
|
||||||
|
NORMALIZE_INPUT_PATH=""
|
||||||
|
NORMALIZE_OUTPUT_PATH="$DEFAULT_NORMALIZED_BACKUP_PATH"
|
||||||
|
NORMALIZE_ROLE="$(id -un)"
|
||||||
|
NORMALIZE_DATABASE="${PG_DATABASE:-testdb}"
|
||||||
|
NORMALIZE_ADMIN_ROLE="postgres"
|
||||||
|
NORMALIZE_PORT="${PG_PORT:-5432}"
|
||||||
|
NORMALIZE_PRELOAD_LIBRARIES="${PG_SHARED_PRELOAD_LIBRARIES:-pg_cron}"
|
||||||
|
NORMALIZE_DATABASE_EXPLICIT=0
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
-h|--help)
|
||||||
|
help_normalize_backup
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-o|--output)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "normalize-backup: --output requires a path"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
NORMALIZE_OUTPUT_PATH="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--role)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "normalize-backup: --role requires a name"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
NORMALIZE_ROLE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--database)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "normalize-backup: --database requires a name"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
NORMALIZE_DATABASE="$2"
|
||||||
|
NORMALIZE_DATABASE_EXPLICIT=1
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--admin-role)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "normalize-backup: --admin-role requires a name"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
NORMALIZE_ADMIN_ROLE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--*|-*)
|
||||||
|
log error "normalize-backup argument $1 does not exist"
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
if [ -n "$NORMALIZE_INPUT_PATH" ]; then
|
||||||
|
log error "normalize-backup: unexpected argument $1"
|
||||||
|
exit 9
|
||||||
|
fi
|
||||||
|
NORMALIZE_INPUT_PATH="$1"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$NORMALIZE_INPUT_PATH" ]; then
|
||||||
|
NORMALIZE_INPUT_PATH="$DEFAULT_BACKUP_PATH"
|
||||||
|
fi
|
||||||
|
if [ -z "$NORMALIZE_ROLE" ] || [ -z "$NORMALIZE_DATABASE" ] || [ -z "$NORMALIZE_ADMIN_ROLE" ]; then
|
||||||
|
log error "normalize-backup: role, database, and admin role must be non-empty"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
if [ ! -d "$NORMALIZE_INPUT_PATH" ]; then
|
||||||
|
log error "normalize-backup: input backup directory not found: $NORMALIZE_INPUT_PATH"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
if [ ! -f "$NORMALIZE_INPUT_PATH/base.tar.gz" ]; then
|
||||||
|
log error "normalize-backup: missing $NORMALIZE_INPUT_PATH/base.tar.gz"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
NORMALIZE_INPUT_ABS=$(realpath "$NORMALIZE_INPUT_PATH")
|
||||||
|
NORMALIZE_OUTPUT_ABS=$(realpath -m "$NORMALIZE_OUTPUT_PATH")
|
||||||
|
|
||||||
|
if [ "$NORMALIZE_INPUT_ABS" = "$NORMALIZE_OUTPUT_ABS" ]; then
|
||||||
|
log error "normalize-backup: input and output paths must differ"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
case "$NORMALIZE_OUTPUT_ABS/" in
|
||||||
|
"$NORMALIZE_INPUT_ABS"/*)
|
||||||
|
log error "normalize-backup: output path must not be inside input backup"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
case "$NORMALIZE_INPUT_ABS/" in
|
||||||
|
"$NORMALIZE_OUTPUT_ABS"/*)
|
||||||
|
log error "normalize-backup: input backup must not be inside output path"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
mkdir -p "$(dirname "$NORMALIZE_OUTPUT_ABS")"
|
||||||
|
|
||||||
|
NORMALIZE_TMPDIR=$(mktemp -d)
|
||||||
|
NORMALIZE_PGDATA="$NORMALIZE_TMPDIR/data"
|
||||||
|
NORMALIZE_SOCKDIR="$NORMALIZE_TMPDIR/sock"
|
||||||
|
NORMALIZE_LOG="$NORMALIZE_TMPDIR/postgres.log"
|
||||||
|
trap '___normalize_backup_cleanup "$NORMALIZE_TMPDIR" "$NORMALIZE_PGDATA"' EXIT INT HUP
|
||||||
|
|
||||||
|
mkdir -m 700 "$NORMALIZE_PGDATA"
|
||||||
|
mkdir -p "$NORMALIZE_SOCKDIR" "$NORMALIZE_PGDATA/pg_wal"
|
||||||
|
|
||||||
|
log notice "extracting backup into temporary PGDATA"
|
||||||
|
tar -xzf "$NORMALIZE_INPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA"
|
||||||
|
if [ -f "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" ]; then
|
||||||
|
tar -xzf "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" -C "$NORMALIZE_PGDATA/pg_wal"
|
||||||
|
fi
|
||||||
|
|
||||||
|
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
|
||||||
|
|
||||||
|
if [ ! -f "$NORMALIZE_PGDATA/postgresql.conf" ]; then
|
||||||
|
cat > "$NORMALIZE_PGDATA/postgresql.conf" <<EOF
|
||||||
|
listen_addresses = ''
|
||||||
|
port = $NORMALIZE_PORT
|
||||||
|
unix_socket_directories = '$NORMALIZE_SOCKDIR'
|
||||||
|
logging_collector = off
|
||||||
|
shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
|
||||||
|
cron.database_name = '$NORMALIZE_DATABASE'
|
||||||
|
cron.host = '$NORMALIZE_SOCKDIR'
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
cat > "$NORMALIZE_PGDATA/pg_hba.conf" <<EOF
|
||||||
|
local all all trust
|
||||||
|
EOF
|
||||||
|
: > "$NORMALIZE_PGDATA/pg_ident.conf"
|
||||||
|
cat > "$NORMALIZE_PGDATA/postgresql.auto.conf" <<EOF
|
||||||
|
# Local-dev normalized backup overrides.
|
||||||
|
listen_addresses = ''
|
||||||
|
port = '$NORMALIZE_PORT'
|
||||||
|
unix_socket_directories = '$NORMALIZE_SOCKDIR'
|
||||||
|
logging_collector = 'off'
|
||||||
|
hba_file = '$NORMALIZE_PGDATA/pg_hba.conf'
|
||||||
|
ident_file = '$NORMALIZE_PGDATA/pg_ident.conf'
|
||||||
|
shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
|
||||||
|
cron.database_name = '$NORMALIZE_DATABASE'
|
||||||
|
cron.host = '$NORMALIZE_SOCKDIR'
|
||||||
|
EOF
|
||||||
|
|
||||||
|
log notice "starting temporary local cluster"
|
||||||
|
with_closed_fds pg_ctl -D "$NORMALIZE_PGDATA" -o "-F" -w start > "$NORMALIZE_LOG" 2>&1 || {
|
||||||
|
log error "normalize-backup: failed to start temporary cluster; see $NORMALIZE_LOG"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" postgres
|
||||||
|
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template1
|
||||||
|
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template0
|
||||||
|
|
||||||
|
if [ "$NORMALIZE_DATABASE_EXPLICIT" -eq 0 ]; then
|
||||||
|
detected_database=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||||
|
-tAc "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate AND datname <> 'postgres' ORDER BY oid LIMIT 1" 2>/dev/null | sed '/^$/d' | head -n 1)
|
||||||
|
if [ -n "$detected_database" ]; then
|
||||||
|
NORMALIZE_DATABASE="$detected_database"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
NORMALIZE_ROLE_SQL=$(___sql_literal "$NORMALIZE_ROLE")
|
||||||
|
psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||||
|
-v ON_ERROR_STOP=1 -c "DO \$\$ DECLARE v_role text := $NORMALIZE_ROLE_SQL; BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = v_role) THEN EXECUTE format('CREATE ROLE %I LOGIN SUPERUSER', v_role); ELSE EXECUTE format('ALTER ROLE %I LOGIN SUPERUSER', v_role); END IF; END \$\$;"
|
||||||
|
|
||||||
|
NORMALIZE_DATABASE_SQL=$(___sql_literal "$NORMALIZE_DATABASE")
|
||||||
|
NORMALIZE_DATABASE_EXISTS=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||||
|
-tAc "SELECT 1 FROM pg_database WHERE datname = $NORMALIZE_DATABASE_SQL" 2>/dev/null || true)
|
||||||
|
if [ "$NORMALIZE_DATABASE_EXISTS" != "1" ]; then
|
||||||
|
createdb -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" \
|
||||||
|
-O "$NORMALIZE_ROLE" "$NORMALIZE_DATABASE"
|
||||||
|
fi
|
||||||
|
___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" "$NORMALIZE_DATABASE"
|
||||||
|
|
||||||
|
log notice "stopping temporary local cluster"
|
||||||
|
pg_ctl -D "$NORMALIZE_PGDATA" -m fast -w stop >> "$NORMALIZE_LOG" 2>&1
|
||||||
|
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
|
||||||
|
|
||||||
|
log notice "writing normalized backup to $WHITE$NORMALIZE_OUTPUT_ABS$NC"
|
||||||
|
rm -rf "$NORMALIZE_OUTPUT_ABS"
|
||||||
|
mkdir -p "$NORMALIZE_OUTPUT_ABS"
|
||||||
|
tar -czf "$NORMALIZE_OUTPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA" .
|
||||||
|
printf '%s\n' "$NORMALIZE_DATABASE" > "$NORMALIZE_OUTPUT_ABS/database_name"
|
||||||
|
|
||||||
|
___normalize_backup_cleanup "$NORMALIZE_TMPDIR" ""
|
||||||
|
trap - EXIT INT HUP
|
||||||
|
restore_namespace
|
||||||
|
}
|
||||||
|
|
||||||
subcommand_log() {
|
subcommand_log() {
|
||||||
change_namespace 'db log'
|
change_namespace 'db log'
|
||||||
: "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}"
|
: "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}"
|
||||||
@@ -1175,6 +1456,7 @@ ___diff_dump_schema() {
|
|||||||
local port="$2"
|
local port="$2"
|
||||||
local output_file="$3"
|
local output_file="$3"
|
||||||
local tables="${4:-}"
|
local tables="${4:-}"
|
||||||
|
local database_name="${5:-${PG_DATABASE:-testdb}}"
|
||||||
|
|
||||||
log info "dumping schema to $WHITE$output_file$NC"
|
log info "dumping schema to $WHITE$output_file$NC"
|
||||||
|
|
||||||
@@ -1189,17 +1471,17 @@ ___diff_dump_schema() {
|
|||||||
IFS="$old_IFS"
|
IFS="$old_IFS"
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
pg_dump -h "$socket_dir" -p "$port" testdb \
|
pg_dump -h "$socket_dir" -p "$port" "$database_name" \
|
||||||
--schema-only --no-owner --no-privileges \
|
--schema-only --no-owner --no-privileges \
|
||||||
$table_args > "$output_file" 2>/dev/null
|
$table_args > "$output_file" 2>/dev/null
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
pg_dump -h "$socket_dir" -p "$port" testdb \
|
pg_dump -h "$socket_dir" -p "$port" "$database_name" \
|
||||||
--data-only --no-owner --no-privileges \
|
--data-only --no-owner --no-privileges \
|
||||||
$table_args >> "$output_file" 2>/dev/null
|
$table_args >> "$output_file" 2>/dev/null
|
||||||
else
|
else
|
||||||
# Schema only
|
# Schema only
|
||||||
pg_dump -h "$socket_dir" -p "$port" testdb \
|
pg_dump -h "$socket_dir" -p "$port" "$database_name" \
|
||||||
--schema-only --no-owner --no-privileges \
|
--schema-only --no-owner --no-privileges \
|
||||||
> "$output_file" 2>/dev/null
|
> "$output_file" 2>/dev/null
|
||||||
fi
|
fi
|
||||||
@@ -1208,7 +1490,8 @@ ___diff_dump_schema() {
|
|||||||
___diff_immutable_tables() {
|
___diff_immutable_tables() {
|
||||||
local socket_dir="$1"
|
local socket_dir="$1"
|
||||||
local port="$2"
|
local port="$2"
|
||||||
psql -h "$socket_dir" -p "$port" -d testdb -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
|
local database_name="${3:-${PG_DATABASE:-testdb}}"
|
||||||
|
psql -h "$socket_dir" -p "$port" -d "$database_name" -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
|
||||||
SELECT n.nspname || '.' || c.relname
|
SELECT n.nspname || '.' || c.relname
|
||||||
FROM pg_inherits i
|
FROM pg_inherits i
|
||||||
JOIN pg_class c ON c.oid = i.inhrelid
|
JOIN pg_class c ON c.oid = i.inhrelid
|
||||||
@@ -1226,8 +1509,9 @@ ___diff_immutable_data() {
|
|||||||
local sock2="$3"
|
local sock2="$3"
|
||||||
local port2="$4"
|
local port2="$4"
|
||||||
local out_file="$5"
|
local out_file="$5"
|
||||||
|
local database_name="${6:-${PG_DATABASE:-testdb}}"
|
||||||
|
|
||||||
if ! psql -h "$sock1" -p "$port1" -d testdb -tAc \
|
if ! psql -h "$sock1" -p "$port1" -d "$database_name" -tAc \
|
||||||
"SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='hectic' AND c.relname='immutable';" \
|
"SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='hectic' AND c.relname='immutable';" \
|
||||||
>/dev/null 2>&1
|
>/dev/null 2>&1
|
||||||
then
|
then
|
||||||
@@ -1256,10 +1540,10 @@ ___diff_immutable_data() {
|
|||||||
log info " $tbl"
|
log info " $tbl"
|
||||||
: > "$data1"
|
: > "$data1"
|
||||||
: > "$data2"
|
: > "$data2"
|
||||||
pg_dump -h "$sock1" -p "$port1" testdb \
|
pg_dump -h "$sock1" -p "$port1" "$database_name" \
|
||||||
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
|
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
|
||||||
> "$data1" 2>/dev/null || :
|
> "$data1" 2>/dev/null || :
|
||||||
pg_dump -h "$sock2" -p "$port2" testdb \
|
pg_dump -h "$sock2" -p "$port2" "$database_name" \
|
||||||
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
|
--data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
|
||||||
> "$data2" 2>/dev/null || :
|
> "$data2" 2>/dev/null || :
|
||||||
{
|
{
|
||||||
@@ -1405,6 +1689,8 @@ subcommand_diff() {
|
|||||||
DIFF_TABLES="" # TODO: add cron table
|
DIFF_TABLES="" # TODO: add cron table
|
||||||
DIFF_NO_CRON=0 # TODO: useless option
|
DIFF_NO_CRON=0 # TODO: useless option
|
||||||
DIFF_BACKUP_PATH=""
|
DIFF_BACKUP_PATH=""
|
||||||
|
DIFF_IGNORE_MIGRATION_FAIL=0
|
||||||
|
DIFF_DATABASE="${PG_DATABASE:-testdb}"
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case $1 in
|
case $1 in
|
||||||
@@ -1420,6 +1706,10 @@ subcommand_diff() {
|
|||||||
DIFF_NO_CRON=1
|
DIFF_NO_CRON=1
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
|
--ignore-migration-fail)
|
||||||
|
DIFF_IGNORE_MIGRATION_FAIL=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
-m|--mock)
|
-m|--mock)
|
||||||
HYDRATE_USE_MOCK=1
|
HYDRATE_USE_MOCK=1
|
||||||
shift
|
shift
|
||||||
@@ -1433,9 +1723,8 @@ subcommand_diff() {
|
|||||||
exit 9
|
exit 9
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
# NOTE: yes, RESTORE, not DIFF prefix
|
if [ -z "$DIFF_BACKUP_PATH" ]; then
|
||||||
if [ -z "$RESTORE_BACKUP_PATH" ]; then
|
DIFF_BACKUP_PATH="$1"
|
||||||
RESTORE_BACKUP_PATH="$1"
|
|
||||||
shift
|
shift
|
||||||
else
|
else
|
||||||
log error "diff: unexpected argument $1"
|
log error "diff: unexpected argument $1"
|
||||||
@@ -1445,14 +1734,21 @@ subcommand_diff() {
|
|||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if [ -z "$DIFF_BACKUP_PATH" ]; then
|
||||||
|
DIFF_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
|
||||||
|
fi
|
||||||
|
if [ -f "$DIFF_BACKUP_PATH/database_name" ]; then
|
||||||
|
DIFF_DATABASE=$(tr -d '\n' < "$DIFF_BACKUP_PATH/database_name")
|
||||||
|
fi
|
||||||
|
|
||||||
DIFF_TMPDIR="${LOCAL_DIR}/focus/database-diff-operation"
|
DIFF_TMPDIR="${LOCAL_DIR}/focus/database-diff-operation"
|
||||||
DIFF_PGDATA1="$DIFF_TMPDIR/pgdata1"
|
DIFF_PGDATA1="$DIFF_TMPDIR/pgdata1"
|
||||||
DIFF_PGDATA2="$DIFF_TMPDIR/pgdata2"
|
DIFF_PGDATA2="$DIFF_TMPDIR/pgdata2"
|
||||||
# TODO: suka, logi drugie
|
# TODO: suka, logi drugie
|
||||||
DIFF_PGLOGFILE1="$DIFF_PGDATA1/logfile"
|
DIFF_PGLOGFILE1="$DIFF_PGDATA1/logfile"
|
||||||
DIFF_PGLOGFILE2="$DIFF_PGDATA2/logfile"
|
DIFF_PGLOGFILE2="$DIFF_PGDATA2/logfile"
|
||||||
DIFF_PGURL1="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA1/sock"
|
DIFF_PGURL1="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA1/sock"
|
||||||
DIFF_PGURL2="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA2/sock"
|
DIFF_PGURL2="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA2/sock"
|
||||||
|
|
||||||
if [ "${DIFF_LOG+x}" ]; then
|
if [ "${DIFF_LOG+x}" ]; then
|
||||||
log_pager -O "$DIFF_PGLOGFILE1" "$DIFF_PGLOGFILE2"
|
log_pager -O "$DIFF_PGLOGFILE1" "$DIFF_PGLOGFILE2"
|
||||||
@@ -1474,21 +1770,26 @@ subcommand_diff() {
|
|||||||
|
|
||||||
log notice "provisioning ${WHITE}DB1$NC (backup + migrations)"
|
log notice "provisioning ${WHITE}DB1$NC (backup + migrations)"
|
||||||
|
|
||||||
PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore
|
PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore "$DIFF_BACKUP_PATH"
|
||||||
|
|
||||||
log info "applying migrations to ${WHITE}DB1$NC"
|
log info "applying migrations to ${WHITE}DB1$NC"
|
||||||
subcommand_migrator migrate up all \
|
subcommand_migrator migrate up all \
|
||||||
--db-url \
|
--db-url \
|
||||||
"$DIFF_PGURL1" \
|
"$DIFF_PGURL1" \
|
||||||
|| {
|
|| {
|
||||||
log warn "migrations failed or none to apply"
|
if [ "$DIFF_IGNORE_MIGRATION_FAIL" = "1" ]; then
|
||||||
|
log warn "migrations failed; continuing because --ignore-migration-fail is set"
|
||||||
|
else
|
||||||
|
log error "migrations failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
log notice "provisioning ${WHITE}DB2$NC (current sources)"
|
log notice "provisioning ${WHITE}DB2$NC (current sources)"
|
||||||
|
|
||||||
log info "initializing ${WHITE}DB2$NC with postgres-init"
|
log info "initializing ${WHITE}DB2$NC with postgres-init"
|
||||||
PG_WORKING_DIR="$DIFF_PGDATA2" \
|
PG_WORKING_DIR="$DIFF_PGDATA2" \
|
||||||
PG_DATABASE="testdb" \
|
PG_DATABASE="$DIFF_DATABASE" \
|
||||||
PG_DISABLE_LOGGING=1 \
|
PG_DISABLE_LOGGING=1 \
|
||||||
postgres-init || {
|
postgres-init || {
|
||||||
log error "failed to initialize ${WHITE}DB2$NC"
|
log error "failed to initialize ${WHITE}DB2$NC"
|
||||||
@@ -1514,8 +1815,11 @@ subcommand_diff() {
|
|||||||
DIFF_DUMP1="$DIFF_TMPDIR/target.sql"
|
DIFF_DUMP1="$DIFF_TMPDIR/target.sql"
|
||||||
DIFF_DUMP2="$DIFF_TMPDIR/source.sql"
|
DIFF_DUMP2="$DIFF_TMPDIR/source.sql"
|
||||||
|
|
||||||
___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES"
|
___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES" "$DIFF_DATABASE"
|
||||||
___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES"
|
___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES" "$DIFF_DATABASE"
|
||||||
|
|
||||||
|
sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP1" || exit 1
|
||||||
|
sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP2" || exit 1
|
||||||
|
|
||||||
# Optional: filter out cron tables
|
# Optional: filter out cron tables
|
||||||
if [ "$DIFF_NO_CRON" = "1" ]; then
|
if [ "$DIFF_NO_CRON" = "1" ]; then
|
||||||
@@ -1541,7 +1845,8 @@ subcommand_diff() {
|
|||||||
___diff_immutable_data \
|
___diff_immutable_data \
|
||||||
"$DIFF_PGDATA1/sock" "5432" \
|
"$DIFF_PGDATA1/sock" "5432" \
|
||||||
"$DIFF_PGDATA2/sock" "5432" \
|
"$DIFF_PGDATA2/sock" "5432" \
|
||||||
"$DIFF_TMPDIR/diff"
|
"$DIFF_TMPDIR/diff" \
|
||||||
|
"$DIFF_DATABASE"
|
||||||
data_status=$?
|
data_status=$?
|
||||||
|
|
||||||
if [ "$schema_differs" = 0 ] && [ "$data_status" = 0 ]; then
|
if [ "$schema_differs" = 0 ] && [ "$data_status" = 0 ]; then
|
||||||
@@ -1573,6 +1878,14 @@ if ! [ "${AS_LIBRARY+x}" ]; then
|
|||||||
DB_URL=$2
|
DB_URL=$2
|
||||||
shift 2
|
shift 2
|
||||||
;;
|
;;
|
||||||
|
normalize-backup)
|
||||||
|
if [ "${SUBCOMMAND+x}" ]; then
|
||||||
|
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
||||||
|
else
|
||||||
|
SUBCOMMAND="normalize_backup"
|
||||||
|
fi
|
||||||
|
shift
|
||||||
|
;;
|
||||||
deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init)
|
deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init)
|
||||||
if [ "${SUBCOMMAND+x}" ]; then
|
if [ "${SUBCOMMAND+x}" ]; then
|
||||||
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
# shellcheck shell=dash
|
||||||
|
|
||||||
|
: "${SCRIPT_NAME:=$(basename "$0")}"
|
||||||
|
|
||||||
|
help() {
|
||||||
|
# shellcheck disable=SC2059
|
||||||
|
printf "$(cat <<EOF
|
||||||
|
${BGREEN}Usage:${NC} $SCRIPT_NAME [OPTIONS] <SUBCOMMAND> [OPTIONS]
|
||||||
|
|
||||||
|
PostgreSQL operations utility.
|
||||||
|
|
||||||
|
${BGREEN}Global Options:${NC}
|
||||||
|
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
|
||||||
|
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
|
||||||
|
|
||||||
|
${BGREEN}Subcommands:${NC}
|
||||||
|
${BCYAN}secrets load${NC} [OPTIONS] Apply hectic bundle and load secrets
|
||||||
|
|
||||||
|
${BGREEN}Environment:${NC}
|
||||||
|
${BBLACK}PGURL${NC} PostgreSQL connection string fallback
|
||||||
|
${BBLACK}DB_URL${NC} PostgreSQL connection string fallback
|
||||||
|
${BBLACK}HECTIC_DOTENV_CONTENT${NC} Raw dotenv content to load
|
||||||
|
${BBLACK}HECTIC_DOTENV_FILE${NC} Dotenv file to read when readable
|
||||||
|
${BBLACK}LOCAL_DIR${NC} Used with ENVIRONMENT for .env fallback
|
||||||
|
${BBLACK}ENVIRONMENT${NC} Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
)"
|
||||||
|
}
|
||||||
|
|
||||||
|
help_secrets_load() {
|
||||||
|
# shellcheck disable=SC2059
|
||||||
|
printf "$(cat <<EOF
|
||||||
|
${BGREEN}Usage:${NC} $SCRIPT_NAME ${BCYAN}secrets load${NC} [OPTIONS]
|
||||||
|
|
||||||
|
Apply the hectic SQL bundle and load dotenv-backed secrets into
|
||||||
|
${BBLACK}hectic.secret${NC}.
|
||||||
|
|
||||||
|
${BGREEN}Options:${NC}
|
||||||
|
${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message
|
||||||
|
${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url> PostgreSQL connection string
|
||||||
|
${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} <path> Dotenv file path
|
||||||
|
|
||||||
|
${BGREEN}Resolution order:${NC}
|
||||||
|
1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC}
|
||||||
|
2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC}
|
||||||
|
3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
|
||||||
|
|
||||||
|
Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv
|
||||||
|
source can be resolved.
|
||||||
|
|
||||||
|
EOF
|
||||||
|
)"
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_pgurl() {
|
||||||
|
if [ -n "${PGURL:-}" ]; then
|
||||||
|
printf '%s' "$PGURL"
|
||||||
|
elif [ -n "${DB_URL:-}" ]; then
|
||||||
|
printf '%s' "$DB_URL"
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_dotenv_content() {
|
||||||
|
dotenv_file="${1:-}"
|
||||||
|
|
||||||
|
if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then
|
||||||
|
printf '%s' "$HECTIC_DOTENV_CONTENT"
|
||||||
|
elif [ -n "$dotenv_file" ]; then
|
||||||
|
if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
cat "$dotenv_file"
|
||||||
|
elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
|
||||||
|
cat "${LOCAL_DIR}/.env.${ENVIRONMENT}"
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
subcommand_secrets_load() {
|
||||||
|
change_namespace 'db ops secrets load'
|
||||||
|
|
||||||
|
dotenv_file="${HECTIC_DOTENV_FILE:-}"
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
-h|--help)
|
||||||
|
help_secrets_load
|
||||||
|
restore_namespace
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-u|--url)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "missing value for $1"
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
PGURL=$2
|
||||||
|
DB_URL=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-f|--dotenv-file)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "missing value for $1"
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
dotenv_file=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--*|-*)
|
||||||
|
log error "secrets load argument $1 does not exist"
|
||||||
|
restore_namespace
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log error "secrets load subcommand $1 does not exist"
|
||||||
|
restore_namespace
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! pgurl="$(resolve_pgurl)"; then
|
||||||
|
log error "PGURL or DB_URL is required"
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then
|
||||||
|
:
|
||||||
|
else
|
||||||
|
dotenv_content_exit_code=$?
|
||||||
|
if [ "$dotenv_content_exit_code" -eq 2 ]; then
|
||||||
|
log error "dotenv file is not readable: $dotenv_file"
|
||||||
|
else
|
||||||
|
log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)"
|
||||||
|
fi
|
||||||
|
restore_namespace
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
log notice "apply hectic bundle and load secrets"
|
||||||
|
apply_hectic_bundle "$pgurl" "$dotenv_content"
|
||||||
|
restore_namespace
|
||||||
|
}
|
||||||
|
|
||||||
|
subcommand_secrets() {
|
||||||
|
change_namespace 'db ops secrets'
|
||||||
|
|
||||||
|
if [ $# -eq 0 ]; then
|
||||||
|
help_secrets_load
|
||||||
|
restore_namespace
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
subcommand=$1
|
||||||
|
shift
|
||||||
|
|
||||||
|
case $subcommand in
|
||||||
|
load)
|
||||||
|
subcommand_secrets_load "$@"
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
help_secrets_load
|
||||||
|
restore_namespace
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log error "secrets subcommand $subcommand does not exist"
|
||||||
|
restore_namespace
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
-h|--help)
|
||||||
|
help
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-u|--url)
|
||||||
|
if [ $# -lt 2 ]; then
|
||||||
|
log error "missing value for $1"
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
PGURL=$2
|
||||||
|
DB_URL=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--*|-*)
|
||||||
|
log error "argument $1 does not exist"
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
subcommand="${1:-}"
|
||||||
|
|
||||||
|
if [ -z "$subcommand" ]; then
|
||||||
|
help
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
shift
|
||||||
|
|
||||||
|
case $subcommand in
|
||||||
|
secrets)
|
||||||
|
subcommand_secrets "$@"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log error "subcommand $subcommand does not exist"
|
||||||
|
exit 9
|
||||||
|
;;
|
||||||
|
esac
|
||||||
+31
-24
@@ -1,32 +1,14 @@
|
|||||||
{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }:
|
{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }:
|
||||||
let
|
let
|
||||||
shell = "${dash}/bin/dash";
|
shell = "${dash}/bin/dash";
|
||||||
|
|
||||||
hecticInheritanceSqlPath = ../../lib/hook/sql/hectic-inheritance.sql;
|
|
||||||
|
|
||||||
hecticInheritance = runCommand "hectic-inheritance" { } ''
|
hecticInheritance = runCommand "hectic-inheritance" { } ''
|
||||||
mkdir -p "$out/share/hectic"
|
mkdir -p "$out/share/hectic"
|
||||||
cp ${hecticInheritanceSqlPath} "$out/share/hectic/hectic-inheritance.sql"
|
cp ${self.lib.hectic.inheritance.path} "$out/share/hectic/hectic-inheritance.sql"
|
||||||
'';
|
'';
|
||||||
|
|
||||||
# Materialize the templated version SQL into the Nix store as a real file
|
applyBundle = self.lib.hectic.applyBundleScript;
|
||||||
# so it can be passed by path to psql -f (alongside the static siblings).
|
|
||||||
hecticVersionSqlFile = pkgs-writeText "hectic-version.sql" self.lib.hectic.version.sql;
|
|
||||||
pkgs-writeText = name: text: runCommand name { inherit text; passAsFile = [ "text" ]; } ''
|
|
||||||
cp "$textPath" "$out"
|
|
||||||
'';
|
|
||||||
|
|
||||||
hecticEnv = ''
|
mkDbDev =
|
||||||
HECTIC_VERSION_SQL=${hecticVersionSqlFile}
|
|
||||||
HECTIC_SECRET_SQL=${self.lib.hectic.secret.path}
|
|
||||||
HECTIC_MIGRATION_SQL=${self.lib.hectic.migration.path}
|
|
||||||
HECTIC_INHERITANCE_SQL=${self.lib.hectic.inheritance.path}
|
|
||||||
export HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL
|
|
||||||
'';
|
|
||||||
|
|
||||||
applyBundle = builtins.readFile self.lib.hectic.applyBundleScript;
|
|
||||||
|
|
||||||
mkDatabase =
|
|
||||||
{ postgresql ? postgresql_17 }:
|
{ postgresql ? postgresql_17 }:
|
||||||
hectic.writeShellApplication {
|
hectic.writeShellApplication {
|
||||||
inherit shell;
|
inherit shell;
|
||||||
@@ -44,9 +26,8 @@ let
|
|||||||
${builtins.readFile hectic.helpers.posix-shell.quote}
|
${builtins.readFile hectic.helpers.posix-shell.quote}
|
||||||
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
|
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
|
||||||
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
|
${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
|
||||||
${hecticEnv}
|
|
||||||
${applyBundle}
|
${applyBundle}
|
||||||
${builtins.readFile ./database.sh}
|
${builtins.readFile ./db-dev.sh}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
@@ -55,6 +36,31 @@ let
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mkDbOps =
|
||||||
|
{ postgresql ? postgresql_17 }:
|
||||||
|
hectic.writeShellApplication {
|
||||||
|
inherit shell;
|
||||||
|
bashOptions = [
|
||||||
|
"errexit"
|
||||||
|
"nounset"
|
||||||
|
];
|
||||||
|
excludeShellChecks = [ "SC2209" ];
|
||||||
|
name = "db-ops";
|
||||||
|
runtimeInputs = [ postgresql coreutils ];
|
||||||
|
|
||||||
|
text = ''
|
||||||
|
${builtins.readFile hectic.helpers.posix-shell.log}
|
||||||
|
${builtins.readFile hectic.helpers.posix-shell.change_namespace}
|
||||||
|
${applyBundle}
|
||||||
|
${builtins.readFile ./db-ops.sh}
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "PostgreSQL operations utility";
|
||||||
|
mainProgram = "db-ops";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
mkPostgresInit =
|
mkPostgresInit =
|
||||||
{ postgresql ? postgresql_17 }:
|
{ postgresql ? postgresql_17 }:
|
||||||
hectic.writeShellApplication {
|
hectic.writeShellApplication {
|
||||||
@@ -92,7 +98,8 @@ let
|
|||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
"db-tool" = lib.makeOverridable mkDatabase { };
|
"db-dev" = lib.makeOverridable mkDbDev { };
|
||||||
|
"db-ops" = lib.makeOverridable mkDbOps { };
|
||||||
"postgres-init" = lib.makeOverridable mkPostgresInit { };
|
"postgres-init" = lib.makeOverridable mkPostgresInit { };
|
||||||
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
|
"postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { };
|
||||||
"hectic-inheritance" = hecticInheritance;
|
"hectic-inheritance" = hecticInheritance;
|
||||||
|
|||||||
@@ -1,9 +1,46 @@
|
|||||||
#!/bin/dash
|
#!/bin/dash
|
||||||
|
|
||||||
|
# Stop a local PostgreSQL cluster started by postgres-init.
|
||||||
|
#
|
||||||
|
# Public contract:
|
||||||
|
# - Accepts PG_WORKING_DIR directly, or derives it from LOCAL_DIR.
|
||||||
|
# - If no cluster root can be resolved, exits successfully without doing
|
||||||
|
# anything. This keeps cleanup safe in traps and partial-init flows.
|
||||||
|
# - If no postmaster.pid exists, treats the cluster as already stopped.
|
||||||
|
# - NO_TTY=1 redirects pg_ctl output into a temp log file instead of writing to
|
||||||
|
# the current terminal.
|
||||||
|
|
||||||
|
postgres_cleanup_help() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: postgres-cleanup
|
||||||
|
|
||||||
|
Stop a local PostgreSQL cluster if it is running.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
PG_WORKING_DIR Cluster root directory
|
||||||
|
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
|
||||||
|
NO_TTY Redirect pg_ctl output to a temp file
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- Returns success if the cluster directory cannot be resolved.
|
||||||
|
- Returns success if postmaster.pid is already absent.
|
||||||
|
- Ignores pg_ctl stop errors so cleanup remains trap-safe.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
postgres_cleanup_main() {
|
postgres_cleanup_main() {
|
||||||
|
case "${1:-}" in
|
||||||
|
-h|--help)
|
||||||
|
postgres_cleanup_help
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
|
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi
|
||||||
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
|
: "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}"
|
||||||
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
|
if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then
|
||||||
|
# Cleanup is intentionally forgiving so it can be used in traps after
|
||||||
|
# partial startup failures without masking the real error.
|
||||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||||
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
|
_pg_log="$(mktemp /tmp/postgres-cleanup.XXXXXX.log)"
|
||||||
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
|
pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop > "$_pg_log" 2>&1 || :
|
||||||
|
|||||||
@@ -1,6 +1,58 @@
|
|||||||
#!/bin/dash
|
#!/bin/dash
|
||||||
|
|
||||||
|
# Initialize or reuse a local PostgreSQL cluster for development workflows.
|
||||||
|
#
|
||||||
|
# Public contract:
|
||||||
|
# - Requires either PG_WORKING_DIR or LOCAL_DIR.
|
||||||
|
# - Produces a ready-to-use database and exports:
|
||||||
|
# POSTGRESQL_HOST, POSTGRESQL_PORT, POSTGRESQL_USER, POSTGRESQL_DATABASE,
|
||||||
|
# PGURL, and also the variable named by PG_URL_VAR.
|
||||||
|
# - Reuses an existing cluster only when PG_REUSE is set and PG_VERSION exists.
|
||||||
|
# - If a reused cluster cannot start, it is reinitialized automatically.
|
||||||
|
#
|
||||||
|
# Important knobs:
|
||||||
|
# - PG_WORKING_DIR: cluster root (defaults to $LOCAL_DIR/focus/postgresql)
|
||||||
|
# - PG_DATABASE: database to create/ensure (default: testdb)
|
||||||
|
# - PG_PORT: socket port (default: 5432)
|
||||||
|
# - PG_URL_VAR: alternate URL variable to export in addition to PGURL
|
||||||
|
# - PG_CONF_FILE: base postgresql.conf to copy on fresh init
|
||||||
|
# - PG_SHARED_PRELOAD_LIBRARIES: preload list; empty string disables pg_cron
|
||||||
|
# - PG_DISABLE_LOGGING=1: disable logging_collector in generated config
|
||||||
|
# - NO_TTY=1: redirect pg_ctl stop/start output into temp log files
|
||||||
|
|
||||||
|
postgres_init_help() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: postgres-init
|
||||||
|
|
||||||
|
Initialize or reuse a local PostgreSQL cluster.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
PG_WORKING_DIR Cluster root directory
|
||||||
|
LOCAL_DIR Fallback root used to derive PG_WORKING_DIR
|
||||||
|
PG_DATABASE Database name to create/ensure (default: testdb)
|
||||||
|
PG_PORT PostgreSQL port / unix socket port (default: 5432)
|
||||||
|
PG_URL_VAR Extra URL variable to export alongside PGURL
|
||||||
|
PG_CONF_FILE Base postgresql.conf copied on fresh init only
|
||||||
|
PG_SHARED_PRELOAD_LIBRARIES Preload list; empty disables pg_cron preload
|
||||||
|
PG_DISABLE_LOGGING Set to 1 to disable logging_collector
|
||||||
|
PG_REUSE Reuse existing cluster if PG_VERSION exists
|
||||||
|
NO_TTY Redirect pg_ctl output to temp files
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- Rewrites runtime socket/port/cron settings on every launch.
|
||||||
|
- Creates the target database if missing.
|
||||||
|
- If a reused cluster exists but cannot start, reinitializes it automatically.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
postgres_init_main() {
|
postgres_init_main() {
|
||||||
|
case "${1:-}" in
|
||||||
|
-h|--help)
|
||||||
|
postgres_init_help
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
|
if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then
|
||||||
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
|
printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2
|
||||||
return 1
|
return 1
|
||||||
@@ -25,29 +77,149 @@ postgres_init_main() {
|
|||||||
fi
|
fi
|
||||||
mkdir -p "$sockdir" || return 1
|
mkdir -p "$sockdir" || return 1
|
||||||
|
|
||||||
|
# Reuse is optimistic: if a cluster exists on disk, try to start it first.
|
||||||
|
# We only destroy state after a real startup failure proves the cluster is
|
||||||
|
# broken, which keeps long-lived local DBs intact when possible.
|
||||||
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
|
if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi
|
||||||
|
|
||||||
if [ "$PG_REUSE" -eq 0 ]; then
|
if [ "$PG_REUSE" -eq 0 ]; then
|
||||||
rm -rf "$data" "$sockdir" || return 1
|
rm -rf "$data" "$sockdir" || return 1
|
||||||
mkdir -p "$sockdir" || return 1
|
mkdir -p "$sockdir" || return 1
|
||||||
initdb -D "$data" --no-locale -E UTF8 || return 1
|
initdb -D "$data" --no-locale -E UTF8 || return 1
|
||||||
if [ -n "${PG_CONF_FILE:-}" ]; then
|
if [ -n "${PG_CONF_FILE:-}" ]; then
|
||||||
|
# PG_CONF_FILE replaces the base postgresql.conf on fresh init only. We
|
||||||
|
# still append runtime values later so the helper can relocate sockets,
|
||||||
|
# ports, and cron settings regardless of the custom file contents.
|
||||||
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
||||||
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
||||||
else
|
else
|
||||||
{ printf '%s\n' "listen_addresses = ''"; [ "$PG_DISABLE_LOGGING" -eq 0 ] && { printf '%s\n' 'logging_collector = on'; printf '%s\n' "log_directory = 'log'"; }; [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ] && { printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"; printf '%s\n' "cron.database_name = '$db'"; printf '%s\n' "cron.host = '$sockdir'"; }; :; } >> "$data/postgresql.conf" || return 1
|
{
|
||||||
|
printf '%s\n' "listen_addresses = ''"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' 'logging_collector = on'
|
||||||
|
printf '%s\n' "log_directory = 'log'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.conf" || return 1
|
||||||
fi
|
fi
|
||||||
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Rewrite the runtime knobs on every launch. Reused clusters may have been
|
||||||
|
# started from another workspace/session, so we must override stale socket,
|
||||||
|
# port, and pg_cron wiring before attempting startup.
|
||||||
|
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
||||||
|
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
||||||
|
|
||||||
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||||
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
|
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
|
||||||
|
{
|
||||||
|
printf '%s\n' "port = '$PG_PORT'"
|
||||||
|
printf '%s\n' "unix_socket_directories = '$sockdir'"
|
||||||
|
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
|
||||||
|
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' "logging_collector = 'on'"
|
||||||
|
else
|
||||||
|
printf '%s\n' "logging_collector = 'off'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.auto.conf" || return 1
|
||||||
|
|
||||||
|
_pg_start_exit=0
|
||||||
if [ "${NO_TTY:-0}" = "1" ]; then
|
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||||
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
||||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || return 2
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
|
||||||
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
||||||
else
|
else
|
||||||
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || return 2
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$_pg_start_exit" -ne 0 ]; then
|
||||||
|
if [ "$PG_REUSE" -eq 1 ]; then
|
||||||
|
# Self-heal path: the on-disk cluster exists but cannot complete startup
|
||||||
|
# (for example bad WAL / invalid checkpoint). At this point preserving
|
||||||
|
# the broken state is less useful than reinitializing automatically.
|
||||||
|
printf '%s\n' "postgres-init: reused cluster failed to start; reinitializing $wd" >&2
|
||||||
|
PG_REUSE=0
|
||||||
|
rm -rf "$data" "$sockdir" || return 1
|
||||||
|
mkdir -p "$sockdir" || return 1
|
||||||
|
initdb -D "$data" --no-locale -E UTF8 || return 1
|
||||||
|
if [ -n "${PG_CONF_FILE:-}" ]; then
|
||||||
|
[ -r "$PG_CONF_FILE" ] || { printf '%s\n' "postgres-init: PG_CONF_FILE not readable: $PG_CONF_FILE" >&2; return 1; }
|
||||||
|
cp -f -- "$PG_CONF_FILE" "$data/postgresql.conf" || return 1
|
||||||
|
else
|
||||||
|
{
|
||||||
|
printf '%s\n' "listen_addresses = ''"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' 'logging_collector = on'
|
||||||
|
printf '%s\n' "log_directory = 'log'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.conf" || return 1
|
||||||
|
fi
|
||||||
|
sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1
|
||||||
|
[ -f "$data/postgresql.auto.conf" ] || : > "$data/postgresql.auto.conf"
|
||||||
|
[ -f "$data/pg_ident.conf" ] || : > "$data/pg_ident.conf"
|
||||||
|
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*hba_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*ident_file[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*shared_preload_libraries[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*cron\.database_name[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
sed -i '/^[[:space:]]*cron\.host[[:space:]]*=/d' "$data/postgresql.auto.conf" || return 1
|
||||||
|
{ printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1
|
||||||
|
{
|
||||||
|
printf '%s\n' "port = '$PG_PORT'"
|
||||||
|
printf '%s\n' "unix_socket_directories = '$sockdir'"
|
||||||
|
printf '%s\n' "hba_file = '$data/pg_hba.conf'"
|
||||||
|
printf '%s\n' "ident_file = '$data/pg_ident.conf'"
|
||||||
|
if [ "$PG_DISABLE_LOGGING" -eq 0 ]; then
|
||||||
|
printf '%s\n' "logging_collector = 'on'"
|
||||||
|
else
|
||||||
|
printf '%s\n' "logging_collector = 'off'"
|
||||||
|
fi
|
||||||
|
if [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ]; then
|
||||||
|
printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"
|
||||||
|
printf '%s\n' "cron.database_name = '$db'"
|
||||||
|
printf '%s\n' "cron.host = '$sockdir'"
|
||||||
|
fi
|
||||||
|
} >> "$data/postgresql.auto.conf" || return 1
|
||||||
|
_pg_start_exit=0
|
||||||
|
if [ "${NO_TTY:-0}" = "1" ]; then
|
||||||
|
_pg_log="$(mktemp /tmp/postgres-init-start.XXXXXX.log)"
|
||||||
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start >"$_pg_log" 2>&1 || _pg_start_exit=$?
|
||||||
|
printf '%s\n' "postgres-init: pg_ctl start output redirected to $_pg_log" >&2
|
||||||
|
else
|
||||||
|
with_closed_fds pg_ctl -D "$data" -o "-F" -w start || _pg_start_exit=$?
|
||||||
|
fi
|
||||||
|
[ "$_pg_start_exit" -eq 0 ] || return 2
|
||||||
|
printf '%s\n' "postgres-init: reinitialized broken cluster at $wd" >&2
|
||||||
|
else
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
user="$(id -un)" || return 1
|
user="$(id -un)" || return 1
|
||||||
@@ -61,9 +233,13 @@ postgres_init_main() {
|
|||||||
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
|
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
|
||||||
|
|
||||||
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
|
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
|
||||||
_pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
|
# Export both names for compatibility: some callers consume plain PGURL,
|
||||||
|
# while multi-cluster shells also need a project-specific variable like
|
||||||
|
# WIPE_PGURL or BMSEARCH_PGURL in the same environment.
|
||||||
|
PGURL="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
|
||||||
|
export PGURL
|
||||||
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
|
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac
|
||||||
export "${PG_URL_VAR}=${_pg_url}" || return 1
|
export "${PG_URL_VAR}=${PGURL}" || return 1
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+8
-1
@@ -123,6 +123,7 @@ in {
|
|||||||
py3-swifter = pkgs.callPackage ./py3-swifter.nix {};
|
py3-swifter = pkgs.callPackage ./py3-swifter.nix {};
|
||||||
py3-aiogram-newsletter = pkgs.callPackage ./py3-aiogram-newsletter.nix {};
|
py3-aiogram-newsletter = pkgs.callPackage ./py3-aiogram-newsletter.nix {};
|
||||||
py3-openai-shap-e = pkgs.callPackage ./py3-openai-shap-e.nix {};
|
py3-openai-shap-e = pkgs.callPackage ./py3-openai-shap-e.nix {};
|
||||||
|
hiddify-core = pkgs.callPackage ./hiddify-core {};
|
||||||
nvim-alias = pkgs.callPackage ./nvim-alias.nix {};
|
nvim-alias = pkgs.callPackage ./nvim-alias.nix {};
|
||||||
bolt-unpack = pkgs.callPackage ./bolt-unpack.nix {};
|
bolt-unpack = pkgs.callPackage ./bolt-unpack.nix {};
|
||||||
merge-archive = pkgs.callPackage ./merge-archive {};
|
merge-archive = pkgs.callPackage ./merge-archive {};
|
||||||
@@ -131,6 +132,7 @@ in {
|
|||||||
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
|
github-gh-tl = pkgs.callPackage ./github/gh-tl.nix {};
|
||||||
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
|
supabase-with-env-collection = pkgs.callPackage ./supabase-with-env-collection.nix {};
|
||||||
migration-name = pkgs.callPackage ./migration-name.nix {};
|
migration-name = pkgs.callPackage ./migration-name.nix {};
|
||||||
|
plgo = pkgs.callPackage ./plgo {};
|
||||||
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
|
pg-from = pkgs.callPackage ./postgres/pg-from/default.nix rust.commonArgs;
|
||||||
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
|
pg-schema = pkgs.callPackage ./postgres/pg-schema/default.nix rust.commonArgs;
|
||||||
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
|
pg_wdumpall = pkgs.callPackage ./postgres/pg_wdumpall.nix rust.commonArgs;
|
||||||
@@ -141,14 +143,18 @@ in {
|
|||||||
watch = pkgs.callPackage ./c/watch/default.nix {};
|
watch = pkgs.callPackage ./c/watch/default.nix {};
|
||||||
support-bot = pkgs.callPackage ./support-bot {};
|
support-bot = pkgs.callPackage ./support-bot {};
|
||||||
gitea-heatmap = pkgs.callPackage ./gitea {};
|
gitea-heatmap = pkgs.callPackage ./gitea {};
|
||||||
|
gitea-runner-nix-image = pkgs.callPackage ./gitea-runner-nix-image {};
|
||||||
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
|
nix-derivation-hash = pkgs.callPackage ./nix-derivation-hash {};
|
||||||
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
|
"sentinèlla" = pkgs.callPackage (./. + "/sentinèlla") {};
|
||||||
deploy = pkgs.callPackage ./deploy { inherit inputs; };
|
deploy = pkgs.callPackage ./deploy { inherit inputs; };
|
||||||
|
element-web = pkgs.callPackage ./element-web {};
|
||||||
shellplot = pkgs.callPackage ./shellplot {};
|
shellplot = pkgs.callPackage ./shellplot {};
|
||||||
|
which-country-rs = pkgs.callPackage ./which-country-rs {};
|
||||||
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
|
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
|
||||||
migrator = pkgs.callPackage ./migrator { inherit self; };
|
migrator = pkgs.callPackage ./migrator { inherit self; };
|
||||||
"parse-uri" = pkgs.callPackage ./parse-uri {};
|
"parse-uri" = pkgs.callPackage ./parse-uri {};
|
||||||
"db-tool" = dbToolPkgs."db-tool";
|
"db-dev" = dbToolPkgs."db-dev";
|
||||||
|
"db-ops" = dbToolPkgs."db-ops";
|
||||||
"postgres-init" = dbToolPkgs."postgres-init";
|
"postgres-init" = dbToolPkgs."postgres-init";
|
||||||
"postgres-cleanup" = dbToolPkgs."postgres-cleanup";
|
"postgres-cleanup" = dbToolPkgs."postgres-cleanup";
|
||||||
"hectic-inheritance" = dbToolPkgs."hectic-inheritance";
|
"hectic-inheritance" = dbToolPkgs."hectic-inheritance";
|
||||||
@@ -170,5 +176,6 @@ in {
|
|||||||
pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
|
pg-15-ext-smtp-client = buildSmtpExt pkgs "15";
|
||||||
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
|
pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15";
|
||||||
pg-15-ext-plsh = buildPlShExt pkgs "15";
|
pg-15-ext-plsh = buildPlShExt pkgs "15";
|
||||||
|
stable-video-diffusion-api = pkgs.callPackage ./stable-video-diffusion-api {};
|
||||||
media-browser = pkgs.callPackage ./media-browser {};
|
media-browser = pkgs.callPackage ./media-browser {};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
{
|
||||||
|
lib,
|
||||||
|
stdenv,
|
||||||
|
fetchFromGitHub,
|
||||||
|
jq,
|
||||||
|
nodejs,
|
||||||
|
jitsi-meet,
|
||||||
|
fetchPnpmDeps,
|
||||||
|
pnpm_10,
|
||||||
|
pnpmConfigHook,
|
||||||
|
faketty,
|
||||||
|
config,
|
||||||
|
conf ? config.element-web.conf or { },
|
||||||
|
}:
|
||||||
|
|
||||||
|
let
|
||||||
|
pnpm = pnpm_10;
|
||||||
|
patchDir = ./patches;
|
||||||
|
patches = if builtins.pathExists patchDir then map (name: patchDir + "/${name}") (
|
||||||
|
builtins.filter (name: lib.hasSuffix ".patch" name) (builtins.attrNames (builtins.readDir patchDir))
|
||||||
|
) else [ ];
|
||||||
|
noPhoningHome = {
|
||||||
|
disable_guests = true;
|
||||||
|
};
|
||||||
|
jitsi-meet-override = jitsi-meet.overrideAttrs (previousAttrs: {
|
||||||
|
meta = removeAttrs previousAttrs.meta [ "knownVulnerabilities" ];
|
||||||
|
});
|
||||||
|
element-web-unwrapped = stdenv.mkDerivation (finalAttrs: {
|
||||||
|
pname = "element-web";
|
||||||
|
version = "1.12.20";
|
||||||
|
|
||||||
|
src = fetchFromGitHub {
|
||||||
|
owner = "element-hq";
|
||||||
|
repo = "element-web";
|
||||||
|
tag = "v${finalAttrs.version}";
|
||||||
|
hash = "sha256-pbzuPgKJ0DmrDSTO7ZTDArX+Xr9k/ndAGZvQg2kMTMQ=";
|
||||||
|
};
|
||||||
|
|
||||||
|
#inherit patches; #WIP
|
||||||
|
|
||||||
|
pnpmDeps = fetchPnpmDeps {
|
||||||
|
pname = "element";
|
||||||
|
inherit (finalAttrs) version src;
|
||||||
|
inherit pnpm;
|
||||||
|
fetcherVersion = 3;
|
||||||
|
hash = "sha256-snm7vaHCVX6vYrkmsz5HlYMKx5Ks3K9jvUinkJ41CU0=";
|
||||||
|
};
|
||||||
|
|
||||||
|
nativeBuildInputs = [
|
||||||
|
jq
|
||||||
|
nodejs
|
||||||
|
pnpm
|
||||||
|
pnpmConfigHook
|
||||||
|
faketty
|
||||||
|
];
|
||||||
|
|
||||||
|
buildPhase = ''
|
||||||
|
runHook preBuild
|
||||||
|
|
||||||
|
cd apps/web
|
||||||
|
|
||||||
|
export VERSION=${finalAttrs.version}
|
||||||
|
faketty pnpm run build
|
||||||
|
|
||||||
|
runHook postBuild
|
||||||
|
'';
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
runHook preInstall
|
||||||
|
|
||||||
|
cp -R webapp $out
|
||||||
|
cp ${jitsi-meet-override}/libs/external_api.min.js $out/jitsi_external_api.min.js
|
||||||
|
echo "${finalAttrs.version}" > "$out/version"
|
||||||
|
jq -s '.[0] * $conf' "config.sample.json" --argjson "conf" '${builtins.toJSON noPhoningHome}' > "$out/config.json"
|
||||||
|
|
||||||
|
runHook postInstall
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
description = "Glossy Matrix collaboration client for the web";
|
||||||
|
homepage = "https://element.io/";
|
||||||
|
changelog = "https://github.com/element-hq/element-web/blob/v${finalAttrs.version}/CHANGELOG.md";
|
||||||
|
teams = [ lib.teams.matrix ];
|
||||||
|
license = lib.licenses.agpl3Plus;
|
||||||
|
platforms = lib.platforms.all;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
in
|
||||||
|
if conf == { } then
|
||||||
|
element-web-unwrapped
|
||||||
|
else
|
||||||
|
stdenv.mkDerivation {
|
||||||
|
pname = "${element-web-unwrapped.pname}-wrapped";
|
||||||
|
inherit (element-web-unwrapped) version meta;
|
||||||
|
|
||||||
|
dontUnpack = true;
|
||||||
|
|
||||||
|
nativeBuildInputs = [ jq ];
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
runHook preInstall
|
||||||
|
|
||||||
|
mkdir -p $out
|
||||||
|
ln -s ${element-web-unwrapped}/* $out
|
||||||
|
rm $out/config.json
|
||||||
|
jq -s '.[0] * $conf' "${element-web-unwrapped}/config.json" --argjson "conf" ${lib.escapeShellArg (builtins.toJSON conf)} > "$out/config.json"
|
||||||
|
|
||||||
|
runHook postInstall
|
||||||
|
'';
|
||||||
|
|
||||||
|
passthru = {
|
||||||
|
inherit conf;
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,481 @@
|
|||||||
|
diff --git a/apps/web/src/IConfigOptions.ts b/apps/web/src/IConfigOptions.ts
|
||||||
|
index a3a6a32..c61c24f 100644
|
||||||
|
--- a/apps/web/src/IConfigOptions.ts
|
||||||
|
+++ b/apps/web/src/IConfigOptions.ts
|
||||||
|
@@ -105,6 +105,15 @@ export interface IConfigOptions {
|
||||||
|
show_labs_settings: boolean;
|
||||||
|
features?: Record<string, boolean>; // <FeatureName, EnabledBool>
|
||||||
|
|
||||||
|
+ hectic?: {
|
||||||
|
+ // This local package config intentionally uses the documented camelCase path
|
||||||
|
+ // `hectic.videoMessages.enabled`.
|
||||||
|
+ // eslint-disable-next-line @typescript-eslint/naming-convention
|
||||||
|
+ videoMessages?: {
|
||||||
|
+ enabled?: boolean;
|
||||||
|
+ };
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
/**
|
||||||
|
* Bug report endpoint URL. "local" means the logs should not be uploaded.
|
||||||
|
*/
|
||||||
|
diff --git a/apps/web/src/SdkConfig.ts b/apps/web/src/SdkConfig.ts
|
||||||
|
index 4be6464..9f48743 100644
|
||||||
|
--- a/apps/web/src/SdkConfig.ts
|
||||||
|
+++ b/apps/web/src/SdkConfig.ts
|
||||||
|
@@ -28,6 +28,12 @@ export const DEFAULTS: DeepReadonly<IConfigOptions> = {
|
||||||
|
integrations_ui_url: "https://scalar.vector.im/",
|
||||||
|
integrations_rest_url: "https://scalar.vector.im/api",
|
||||||
|
show_labs_settings: false,
|
||||||
|
+ hectic: {
|
||||||
|
+ // eslint-disable-next-line @typescript-eslint/naming-convention
|
||||||
|
+ videoMessages: {
|
||||||
|
+ enabled: false,
|
||||||
|
+ },
|
||||||
|
+ },
|
||||||
|
force_verification: false,
|
||||||
|
|
||||||
|
jitsi: {
|
||||||
|
diff --git a/apps/web/src/components/views/rooms/MessageComposer.tsx b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||||
|
index 06c843f..eea76b7 100644
|
||||||
|
--- a/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||||
|
+++ b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||||
|
@@ -33,6 +33,7 @@ import ReplyPreview from "./ReplyPreview";
|
||||||
|
import { UserIdentityWarning } from "./UserIdentityWarning";
|
||||||
|
import { UPDATE_EVENT } from "../../../stores/AsyncStore";
|
||||||
|
import VoiceRecordComposerTile from "./VoiceRecordComposerTile";
|
||||||
|
+import VideoRecordComposerTile from "./VideoRecordComposerTile";
|
||||||
|
import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore";
|
||||||
|
import { RecordingState } from "../../../audio/VoiceRecording";
|
||||||
|
import type ResizeNotifier from "../../../utils/ResizeNotifier";
|
||||||
|
@@ -92,6 +93,7 @@ interface IState {
|
||||||
|
composerContent: string;
|
||||||
|
isComposerEmpty: boolean;
|
||||||
|
haveRecording: boolean;
|
||||||
|
+ haveVideoRecording: boolean;
|
||||||
|
recordingTimeLeftSeconds?: number;
|
||||||
|
me?: RoomMember;
|
||||||
|
isMenuOpen: boolean;
|
||||||
|
@@ -113,6 +115,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
private dispatcherRef?: string;
|
||||||
|
private messageComposerInput = createRef<SendMessageComposerClass>();
|
||||||
|
private voiceRecordingButton = createRef<VoiceRecordComposerTile>();
|
||||||
|
+ private videoRecordingButton = createRef<VideoRecordComposerTile>();
|
||||||
|
private ref = createRef<HTMLDivElement>();
|
||||||
|
private instanceId: number;
|
||||||
|
|
||||||
|
@@ -144,6 +147,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
isComposerEmpty: initialComposerContent?.length === 0,
|
||||||
|
composerContent: initialComposerContent,
|
||||||
|
haveRecording: false,
|
||||||
|
+ haveVideoRecording: false,
|
||||||
|
recordingTimeLeftSeconds: undefined, // when set to a number, shows a toast
|
||||||
|
isMenuOpen: false,
|
||||||
|
isStickerPickerOpen: false,
|
||||||
|
@@ -526,6 +530,18 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
+ private onVideoRecordStartClick = (): void => {
|
||||||
|
+ this.videoRecordingButton.current?.onRecordStartEndClick();
|
||||||
|
+
|
||||||
|
+ if (this.context.narrow) {
|
||||||
|
+ this.toggleButtonMenu();
|
||||||
|
+ }
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private onVideoRecordingStateChange = (haveVideoRecording: boolean): void => {
|
||||||
|
+ this.setState({ haveVideoRecording });
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
public render(): React.ReactNode {
|
||||||
|
let leftIcon: false | JSX.Element = false;
|
||||||
|
if (!this.state.isWysiwygLabEnabled) {
|
||||||
|
@@ -561,13 +577,14 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
const menuPosition = this.getMenuPosition();
|
||||||
|
|
||||||
|
const canSendMessages = this.context.canSendMessages && !this.context.tombstone;
|
||||||
|
+ const hasActiveRecording = this.state.haveRecording || this.state.haveVideoRecording;
|
||||||
|
let composer: ReactNode;
|
||||||
|
if (canSendMessages) {
|
||||||
|
if (this.state.isWysiwygLabEnabled && menuPosition) {
|
||||||
|
composer = (
|
||||||
|
<SendWysiwygComposer
|
||||||
|
key="controls_input"
|
||||||
|
- disabled={this.state.haveRecording}
|
||||||
|
+ disabled={hasActiveRecording}
|
||||||
|
onChange={this.onWysiwygChange}
|
||||||
|
onSend={this.sendMessage}
|
||||||
|
isRichTextEnabled={this.state.isRichTextEnabled}
|
||||||
|
@@ -588,7 +605,7 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
relation={this.props.relation}
|
||||||
|
replyToEvent={this.props.replyToEvent}
|
||||||
|
onChange={this.onChange}
|
||||||
|
- disabled={this.state.haveRecording}
|
||||||
|
+ disabled={hasActiveRecording}
|
||||||
|
toggleStickerPickerOpen={this.toggleStickerPickerOpen}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
@@ -608,6 +625,11 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
replyToEvent={this.props.replyToEvent}
|
||||||
|
/>
|
||||||
|
</Tooltip>,
|
||||||
|
+ <VideoRecordComposerTile
|
||||||
|
+ key="controls_video_record"
|
||||||
|
+ ref={this.videoRecordingButton}
|
||||||
|
+ onRecordingStateChange={this.onVideoRecordingStateChange}
|
||||||
|
+ />,
|
||||||
|
);
|
||||||
|
} else if (this.context.tombstone) {
|
||||||
|
const replacementRoomId = this.context.tombstone.getContent()["replacement_room"];
|
||||||
|
@@ -688,12 +710,13 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
{canSendMessages && (
|
||||||
|
<MessageComposerButtons
|
||||||
|
addEmoji={this.addEmoji}
|
||||||
|
- haveRecording={this.state.haveRecording}
|
||||||
|
+ haveRecording={hasActiveRecording}
|
||||||
|
isMenuOpen={this.state.isMenuOpen}
|
||||||
|
isStickerPickerOpen={this.state.isStickerPickerOpen}
|
||||||
|
menuPosition={menuPosition}
|
||||||
|
relation={this.props.relation}
|
||||||
|
onRecordStartEndClick={this.onRecordStartEndClick}
|
||||||
|
+ onVideoRecordStartClick={this.onVideoRecordStartClick}
|
||||||
|
setStickerPickerOpen={this.setStickerPickerOpen}
|
||||||
|
showLocationButton={
|
||||||
|
!window.electron && SettingsStore.getValue(UIFeature.LocationSharing)
|
||||||
|
diff --git a/apps/web/src/components/views/rooms/MessageComposerButtons.tsx b/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
|
||||||
|
index 6f4f5d1..0c1b7ff 100644
|
||||||
|
--- a/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
|
||||||
|
+++ b/apps/web/src/components/views/rooms/MessageComposerButtons.tsx
|
||||||
|
@@ -44,6 +44,8 @@ import { useSettingValue } from "../../../hooks/useSettings";
|
||||||
|
import AccessibleButton, { type ButtonEvent } from "../elements/AccessibleButton";
|
||||||
|
import { useScopedRoomContext } from "../../../contexts/ScopedRoomContext.tsx";
|
||||||
|
import { useRoomUploadViewModel } from "../../../viewmodels/room/RoomUploadViewModel.tsx";
|
||||||
|
+import SdkConfig from "../../../SdkConfig";
|
||||||
|
+import { isVideoMessageRecorderSupported } from "./VideoRecordComposerTile";
|
||||||
|
|
||||||
|
interface IProps {
|
||||||
|
addEmoji: (emoji: string) => boolean;
|
||||||
|
@@ -52,6 +54,7 @@ interface IProps {
|
||||||
|
isStickerPickerOpen: boolean;
|
||||||
|
menuPosition?: MenuProps;
|
||||||
|
onRecordStartEndClick: () => void;
|
||||||
|
+ onVideoRecordStartClick: () => void;
|
||||||
|
relation?: IEventRelation;
|
||||||
|
setStickerPickerOpen: (isStickerPickerOpen: boolean) => void;
|
||||||
|
showLocationButton: boolean;
|
||||||
|
@@ -103,6 +106,7 @@ const MessageComposerButtons: React.FC<IProps> = (props: IProps) => {
|
||||||
|
)),
|
||||||
|
showStickersButton(props),
|
||||||
|
voiceRecordingButton(props, narrow),
|
||||||
|
+ videoRecordingButton(props, narrow),
|
||||||
|
props.showPollsButton ? pollButton(room, props.relation) : null,
|
||||||
|
showLocationButton(props, room, matrixClient),
|
||||||
|
];
|
||||||
|
@@ -122,6 +126,7 @@ const MessageComposerButtons: React.FC<IProps> = (props: IProps) => {
|
||||||
|
moreButtons = [
|
||||||
|
showStickersButton(props),
|
||||||
|
voiceRecordingButton(props, narrow),
|
||||||
|
+ videoRecordingButton(props, narrow),
|
||||||
|
props.showPollsButton ? pollButton(room, props.relation) : null,
|
||||||
|
showLocationButton(props, room, matrixClient),
|
||||||
|
];
|
||||||
|
@@ -203,6 +208,25 @@ function voiceRecordingButton(props: IProps, narrow: boolean): ReactElement | nu
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
+function videoRecordingButton(props: IProps, narrow: boolean): ReactElement | null {
|
||||||
|
+ // The current recorder skeleton follows the voice recorder and stays out of narrow mode.
|
||||||
|
+ if (narrow || SdkConfig.get("hectic")?.videoMessages?.enabled !== true || !isVideoMessageRecorderSupported()) {
|
||||||
|
+ return null;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return (
|
||||||
|
+ <CollapsibleButton
|
||||||
|
+ key="video_message_send"
|
||||||
|
+ className="mx_MessageComposer_button"
|
||||||
|
+ data-testid="video-message-button"
|
||||||
|
+ onClick={props.onVideoRecordStartClick}
|
||||||
|
+ title={_t("composer|video_message_button")}
|
||||||
|
+ >
|
||||||
|
+ <span aria-hidden="true">●</span>
|
||||||
|
+ </CollapsibleButton>
|
||||||
|
+ );
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
function pollButton(room: Room, relation?: IEventRelation): ReactElement {
|
||||||
|
return <PollButton key="polls" room={room} relation={relation} />;
|
||||||
|
}
|
||||||
|
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..c5f0adc
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
@@ -0,0 +1,249 @@
|
||||||
|
+/*
|
||||||
|
+Copyright 2026 Element Creations Ltd.
|
||||||
|
+
|
||||||
|
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
+Please see LICENSE files in the repository root for full details.
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+import React, { type ReactNode } from "react";
|
||||||
|
+import { logger } from "matrix-js-sdk/src/logger";
|
||||||
|
+import { DeleteIcon, SendSolidIcon, StopSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||||
|
+
|
||||||
|
+import { _t } from "../../../languageHandler";
|
||||||
|
+import AccessibleButton from "../elements/AccessibleButton";
|
||||||
|
+
|
||||||
|
+interface IProps {
|
||||||
|
+ onRecordingStateChange: (haveVideoRecording: boolean) => void;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+interface IState {
|
||||||
|
+ error?: string;
|
||||||
|
+ isRecording: boolean;
|
||||||
|
+ previewUrl?: string;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+export function isVideoMessageRecorderSupported(): boolean {
|
||||||
|
+ return (
|
||||||
|
+ typeof window !== "undefined" &&
|
||||||
|
+ typeof MediaRecorder !== "undefined" &&
|
||||||
|
+ typeof navigator !== "undefined" &&
|
||||||
|
+ !!navigator.mediaDevices?.getUserMedia
|
||||||
|
+ );
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * Container tile for rendering the config-gated video message recorder skeleton in the composer.
|
||||||
|
+ */
|
||||||
|
+export default class VideoRecordComposerTile extends React.PureComponent<IProps, IState> {
|
||||||
|
+ private chunks: Blob[] = [];
|
||||||
|
+ private mediaRecorder?: MediaRecorder;
|
||||||
|
+ private stream?: MediaStream;
|
||||||
|
+ private isRequestingMedia = false;
|
||||||
|
+ private isUnmounted = false;
|
||||||
|
+ private shouldDiscardRecording = false;
|
||||||
|
+
|
||||||
|
+ public constructor(props: IProps) {
|
||||||
|
+ super(props);
|
||||||
|
+
|
||||||
|
+ this.state = {
|
||||||
|
+ isRecording: false,
|
||||||
|
+ };
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ public componentWillUnmount(): void {
|
||||||
|
+ this.isUnmounted = true;
|
||||||
|
+ this.disposeRecording();
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ public onRecordStartEndClick = async (): Promise<void> => {
|
||||||
|
+ if (this.state.isRecording) {
|
||||||
|
+ this.stopRecording();
|
||||||
|
+ return;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ await this.startRecording();
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private startRecording = async (): Promise<void> => {
|
||||||
|
+ if (this.isRequestingMedia || this.mediaRecorder) return;
|
||||||
|
+
|
||||||
|
+ if (!isVideoMessageRecorderSupported()) {
|
||||||
|
+ if (!this.isUnmounted) {
|
||||||
|
+ this.setState({ error: _t("composer|video_message_error") });
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ }
|
||||||
|
+ return;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ this.isRequestingMedia = true;
|
||||||
|
+ this.shouldDiscardRecording = false;
|
||||||
|
+ this.revokePreviewUrl();
|
||||||
|
+ this.chunks = [];
|
||||||
|
+
|
||||||
|
+ try {
|
||||||
|
+ const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
|
||||||
|
+ if (this.isUnmounted) {
|
||||||
|
+ stream.getTracks().forEach((track) => track.stop());
|
||||||
|
+ return;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ this.stream = stream;
|
||||||
|
+ const mediaRecorder = new MediaRecorder(stream);
|
||||||
|
+
|
||||||
|
+ mediaRecorder.ondataavailable = (ev: BlobEvent): void => {
|
||||||
|
+ if (ev.data.size > 0) {
|
||||||
|
+ this.chunks.push(ev.data);
|
||||||
|
+ }
|
||||||
|
+ };
|
||||||
|
+ mediaRecorder.onerror = (ev: Event): void => {
|
||||||
|
+ logger.error("Error recording video message:", ev);
|
||||||
|
+ this.setState({ error: _t("composer|video_message_error") });
|
||||||
|
+ };
|
||||||
|
+ mediaRecorder.onstop = this.onRecorderStop;
|
||||||
|
+
|
||||||
|
+ this.mediaRecorder = mediaRecorder;
|
||||||
|
+ mediaRecorder.start();
|
||||||
|
+
|
||||||
|
+ this.setState({ error: undefined, isRecording: true, previewUrl: undefined });
|
||||||
|
+ this.props.onRecordingStateChange(true);
|
||||||
|
+ } catch (e) {
|
||||||
|
+ logger.error("Error starting video message recording:", e);
|
||||||
|
+ this.stopStream();
|
||||||
|
+ if (!this.isUnmounted) {
|
||||||
|
+ this.setState({ error: _t("composer|video_message_error"), isRecording: false, previewUrl: undefined });
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ }
|
||||||
|
+ } finally {
|
||||||
|
+ this.isRequestingMedia = false;
|
||||||
|
+ }
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private stopRecording = (): void => {
|
||||||
|
+ if (!this.mediaRecorder) return;
|
||||||
|
+
|
||||||
|
+ if (this.mediaRecorder.state === "inactive") {
|
||||||
|
+ this.onRecorderStop();
|
||||||
|
+ } else {
|
||||||
|
+ this.mediaRecorder.stop();
|
||||||
|
+ }
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private onRecorderStop = (): void => {
|
||||||
|
+ const shouldDiscardRecording = this.shouldDiscardRecording;
|
||||||
|
+ const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
|
||||||
|
+
|
||||||
|
+ this.mediaRecorder = undefined;
|
||||||
|
+ this.stopStream();
|
||||||
|
+
|
||||||
|
+ if (this.isUnmounted) {
|
||||||
|
+ this.chunks = [];
|
||||||
|
+ return;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if (!hasRecording) {
|
||||||
|
+ this.chunks = [];
|
||||||
|
+ this.setState({ isRecording: false, previewUrl: undefined });
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ return;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "video/webm" });
|
||||||
|
+ const previewUrl = URL.createObjectURL(blob);
|
||||||
|
+
|
||||||
|
+ this.chunks = [];
|
||||||
|
+ this.revokePreviewUrl();
|
||||||
|
+ this.setState({ isRecording: false, previewUrl });
|
||||||
|
+ this.props.onRecordingStateChange(true);
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private onCancel = (): void => {
|
||||||
|
+ this.shouldDiscardRecording = true;
|
||||||
|
+ this.disposeRecording();
|
||||||
|
+ this.setState({ error: undefined, isRecording: false, previewUrl: undefined });
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private disposeRecording(): void {
|
||||||
|
+ this.chunks = [];
|
||||||
|
+ this.stopStream();
|
||||||
|
+ this.revokePreviewUrl();
|
||||||
|
+
|
||||||
|
+ if (this.mediaRecorder) {
|
||||||
|
+ this.mediaRecorder.ondataavailable = null;
|
||||||
|
+ this.mediaRecorder.onerror = null;
|
||||||
|
+ this.mediaRecorder.onstop = null;
|
||||||
|
+
|
||||||
|
+ if (this.mediaRecorder.state !== "inactive") {
|
||||||
|
+ this.mediaRecorder.stop();
|
||||||
|
+ }
|
||||||
|
+ this.mediaRecorder = undefined;
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ private stopStream(): void {
|
||||||
|
+ this.stream?.getTracks().forEach((track) => track.stop());
|
||||||
|
+ this.stream = undefined;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ private revokePreviewUrl(): void {
|
||||||
|
+ if (this.state.previewUrl) {
|
||||||
|
+ URL.revokeObjectURL(this.state.previewUrl);
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ public render(): ReactNode {
|
||||||
|
+ if (!this.state.isRecording && !this.state.previewUrl && !this.state.error) return null;
|
||||||
|
+
|
||||||
|
+ const stopButton = this.state.isRecording ? (
|
||||||
|
+ <AccessibleButton
|
||||||
|
+ className="mx_VoiceRecordComposerTile_stop"
|
||||||
|
+ data-testid="video-message-stop-button"
|
||||||
|
+ onClick={this.onRecordStartEndClick}
|
||||||
|
+ title={_t("composer|stop_video_message")}
|
||||||
|
+ >
|
||||||
|
+ <StopSolidIcon />
|
||||||
|
+ </AccessibleButton>
|
||||||
|
+ ) : null;
|
||||||
|
+
|
||||||
|
+ const sendButton = this.state.previewUrl ? (
|
||||||
|
+ <AccessibleButton
|
||||||
|
+ className="mx_VoiceRecordComposerTile_stop"
|
||||||
|
+ data-testid="video-message-send-button"
|
||||||
|
+ disabled={true}
|
||||||
|
+ onClick={null}
|
||||||
|
+ title={_t("composer|send_video_message")}
|
||||||
|
+ >
|
||||||
|
+ <SendSolidIcon />
|
||||||
|
+ </AccessibleButton>
|
||||||
|
+ ) : null;
|
||||||
|
+
|
||||||
|
+ return (
|
||||||
|
+ <>
|
||||||
|
+ {this.state.error && (
|
||||||
|
+ <span className="mx_VoiceRecordComposerTile_failedState" data-testid="video-message-error">
|
||||||
|
+ {this.state.error}
|
||||||
|
+ </span>
|
||||||
|
+ )}
|
||||||
|
+ <AccessibleButton
|
||||||
|
+ className="mx_VoiceRecordComposerTile_delete"
|
||||||
|
+ data-testid="video-message-cancel-button"
|
||||||
|
+ onClick={this.onCancel}
|
||||||
|
+ title={_t("composer|video_message_cancel")}
|
||||||
|
+ >
|
||||||
|
+ <DeleteIcon />
|
||||||
|
+ </AccessibleButton>
|
||||||
|
+ {stopButton}
|
||||||
|
+ {sendButton}
|
||||||
|
+ {this.state.previewUrl && (
|
||||||
|
+ <video
|
||||||
|
+ aria-label={_t("composer|video_message_preview")}
|
||||||
|
+ className="mx_VoiceMessagePrimaryContainer"
|
||||||
|
+ controls={true}
|
||||||
|
+ data-testid="video-message-preview"
|
||||||
|
+ src={this.state.previewUrl}
|
||||||
|
+ />
|
||||||
|
+ )}
|
||||||
|
+ </>
|
||||||
|
+ );
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json
|
||||||
|
index 4ed51db..e11fe90 100644
|
||||||
|
--- a/apps/web/src/i18n/strings/en_EN.json
|
||||||
|
+++ b/apps/web/src/i18n/strings/en_EN.json
|
||||||
|
@@ -641,8 +641,14 @@
|
||||||
|
"room_upgraded_notice": "This room has been replaced and is no longer active.",
|
||||||
|
"send_button_title": "Send message",
|
||||||
|
"send_button_voice_message": "Send voice message",
|
||||||
|
+ "send_video_message": "Send video message",
|
||||||
|
"send_voice_message": "Send voice message",
|
||||||
|
+ "stop_video_message": "Stop video recording",
|
||||||
|
"stop_voice_message": "Stop recording",
|
||||||
|
+ "video_message_button": "Record video message",
|
||||||
|
+ "video_message_cancel": "Cancel video message",
|
||||||
|
+ "video_message_error": "Unable to record video message",
|
||||||
|
+ "video_message_preview": "Video message preview",
|
||||||
|
"voice_message_button": "Voice Message"
|
||||||
|
},
|
||||||
|
"console_dev_note": "If you know what you're doing, Element is open-source, be sure to check out our GitHub (https://github.com/vector-im/element-web/) and contribute!",
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
diff --git a/apps/web/src/components/views/rooms/MessageComposer.tsx b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||||
|
index eea76b7..3483c28 100644
|
||||||
|
--- a/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||||
|
+++ b/apps/web/src/components/views/rooms/MessageComposer.tsx
|
||||||
|
@@ -629,6 +629,9 @@ export class MessageComposer extends React.Component<IProps, IState> {
|
||||||
|
key="controls_video_record"
|
||||||
|
ref={this.videoRecordingButton}
|
||||||
|
onRecordingStateChange={this.onVideoRecordingStateChange}
|
||||||
|
+ relation={this.props.relation}
|
||||||
|
+ replyToEvent={this.props.replyToEvent}
|
||||||
|
+ room={this.props.room}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
} else if (this.context.tombstone) {
|
||||||
|
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
index c5f0adc..851c64e 100644
|
||||||
|
--- a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
@@ -6,22 +6,39 @@ Please see LICENSE files in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import React, { type ReactNode } from "react";
|
||||||
|
+import { type IEventRelation, type MatrixEvent, type Room } from "matrix-js-sdk/src/matrix";
|
||||||
|
import { logger } from "matrix-js-sdk/src/logger";
|
||||||
|
import { DeleteIcon, SendSolidIcon, StopSolidIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||||
|
|
||||||
|
import { _t } from "../../../languageHandler";
|
||||||
|
+import { MatrixClientPeg } from "../../../MatrixClientPeg";
|
||||||
|
+import ContentMessages from "../../../ContentMessages";
|
||||||
|
import AccessibleButton from "../elements/AccessibleButton";
|
||||||
|
|
||||||
|
interface IProps {
|
||||||
|
onRecordingStateChange: (haveVideoRecording: boolean) => void;
|
||||||
|
+ relation?: IEventRelation;
|
||||||
|
+ replyToEvent?: MatrixEvent;
|
||||||
|
+ room: Room;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface IState {
|
||||||
|
error?: string;
|
||||||
|
isRecording: boolean;
|
||||||
|
+ isSending: boolean;
|
||||||
|
previewUrl?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
+const PREFERRED_VIDEO_MIME_TYPES = ["video/webm;codecs=vp8,opus", "video/webm"];
|
||||||
|
+
|
||||||
|
+function preferredVideoMimeType(): string | undefined {
|
||||||
|
+ for (const mimeType of PREFERRED_VIDEO_MIME_TYPES) {
|
||||||
|
+ if (MediaRecorder.isTypeSupported(mimeType)) {
|
||||||
|
+ return mimeType;
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
export function isVideoMessageRecorderSupported(): boolean {
|
||||||
|
return (
|
||||||
|
typeof window !== "undefined" &&
|
||||||
|
@@ -37,6 +54,7 @@ export function isVideoMessageRecorderSupported(): boolean {
|
||||||
|
export default class VideoRecordComposerTile extends React.PureComponent<IProps, IState> {
|
||||||
|
private chunks: Blob[] = [];
|
||||||
|
private mediaRecorder?: MediaRecorder;
|
||||||
|
+ private recordedFile?: File;
|
||||||
|
private stream?: MediaStream;
|
||||||
|
private isRequestingMedia = false;
|
||||||
|
private isUnmounted = false;
|
||||||
|
@@ -47,6 +65,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
|
||||||
|
this.state = {
|
||||||
|
isRecording: false,
|
||||||
|
+ isSending: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -78,6 +97,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
this.isRequestingMedia = true;
|
||||||
|
this.shouldDiscardRecording = false;
|
||||||
|
this.revokePreviewUrl();
|
||||||
|
+ this.recordedFile = undefined;
|
||||||
|
this.chunks = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
@@ -88,7 +108,8 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
}
|
||||||
|
|
||||||
|
this.stream = stream;
|
||||||
|
- const mediaRecorder = new MediaRecorder(stream);
|
||||||
|
+ const mimeType = preferredVideoMimeType();
|
||||||
|
+ const mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
|
||||||
|
|
||||||
|
mediaRecorder.ondataavailable = (ev: BlobEvent): void => {
|
||||||
|
if (ev.data.size > 0) {
|
||||||
|
@@ -104,13 +125,18 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
this.mediaRecorder = mediaRecorder;
|
||||||
|
mediaRecorder.start();
|
||||||
|
|
||||||
|
- this.setState({ error: undefined, isRecording: true, previewUrl: undefined });
|
||||||
|
+ this.setState({ error: undefined, isRecording: true, isSending: false, previewUrl: undefined });
|
||||||
|
this.props.onRecordingStateChange(true);
|
||||||
|
} catch (e) {
|
||||||
|
logger.error("Error starting video message recording:", e);
|
||||||
|
this.stopStream();
|
||||||
|
if (!this.isUnmounted) {
|
||||||
|
- this.setState({ error: _t("composer|video_message_error"), isRecording: false, previewUrl: undefined });
|
||||||
|
+ this.setState({
|
||||||
|
+ error: _t("composer|video_message_error"),
|
||||||
|
+ isRecording: false,
|
||||||
|
+ isSending: false,
|
||||||
|
+ previewUrl: undefined,
|
||||||
|
+ });
|
||||||
|
this.props.onRecordingStateChange(false);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
@@ -129,6 +155,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
};
|
||||||
|
|
||||||
|
private onRecorderStop = (): void => {
|
||||||
|
+ const mimeType = this.mediaRecorder?.mimeType;
|
||||||
|
const shouldDiscardRecording = this.shouldDiscardRecording;
|
||||||
|
const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
|
||||||
|
|
||||||
|
@@ -142,29 +169,63 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
|
||||||
|
if (!hasRecording) {
|
||||||
|
this.chunks = [];
|
||||||
|
- this.setState({ isRecording: false, previewUrl: undefined });
|
||||||
|
+ this.recordedFile = undefined;
|
||||||
|
+ this.setState({ isRecording: false, isSending: false, previewUrl: undefined });
|
||||||
|
this.props.onRecordingStateChange(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
- const blob = new Blob(this.chunks, { type: this.chunks[0]?.type || "video/webm" });
|
||||||
|
- const previewUrl = URL.createObjectURL(blob);
|
||||||
|
+ const blob = new Blob(this.chunks, { type: mimeType || this.chunks[0]?.type || "video/webm" });
|
||||||
|
+ const file = new File([blob], `video-message-${Date.now()}.webm`, { type: blob.type || "video/webm" });
|
||||||
|
+ const previewUrl = URL.createObjectURL(file);
|
||||||
|
|
||||||
|
this.chunks = [];
|
||||||
|
+ this.recordedFile = file;
|
||||||
|
this.revokePreviewUrl();
|
||||||
|
- this.setState({ isRecording: false, previewUrl });
|
||||||
|
+ this.setState({ isRecording: false, isSending: false, previewUrl });
|
||||||
|
this.props.onRecordingStateChange(true);
|
||||||
|
};
|
||||||
|
|
||||||
|
+ private onSend = async (): Promise<void> => {
|
||||||
|
+ if (!this.recordedFile || this.state.isSending) return;
|
||||||
|
+
|
||||||
|
+ this.setState({ isSending: true });
|
||||||
|
+
|
||||||
|
+ try {
|
||||||
|
+ await ContentMessages.sharedInstance().sendContentToRoom(
|
||||||
|
+ this.recordedFile,
|
||||||
|
+ this.props.room.roomId,
|
||||||
|
+ this.props.relation,
|
||||||
|
+ MatrixClientPeg.safeGet(),
|
||||||
|
+ this.props.replyToEvent,
|
||||||
|
+ );
|
||||||
|
+ this.clearRecording();
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ } catch (e) {
|
||||||
|
+ logger.error("Error sending video message recording:", e);
|
||||||
|
+ if (!this.isUnmounted) {
|
||||||
|
+ this.setState({ error: _t("composer|video_message_error"), isSending: false });
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
private onCancel = (): void => {
|
||||||
|
this.shouldDiscardRecording = true;
|
||||||
|
this.disposeRecording();
|
||||||
|
- this.setState({ error: undefined, isRecording: false, previewUrl: undefined });
|
||||||
|
+ this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
|
||||||
|
this.props.onRecordingStateChange(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
+ private clearRecording(): void {
|
||||||
|
+ this.chunks = [];
|
||||||
|
+ this.recordedFile = undefined;
|
||||||
|
+ this.revokePreviewUrl();
|
||||||
|
+ this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
private disposeRecording(): void {
|
||||||
|
this.chunks = [];
|
||||||
|
+ this.recordedFile = undefined;
|
||||||
|
this.stopStream();
|
||||||
|
this.revokePreviewUrl();
|
||||||
|
|
||||||
|
@@ -209,8 +270,8 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
<AccessibleButton
|
||||||
|
className="mx_VoiceRecordComposerTile_stop"
|
||||||
|
data-testid="video-message-send-button"
|
||||||
|
- disabled={true}
|
||||||
|
- onClick={null}
|
||||||
|
+ disabled={!this.recordedFile || this.state.isSending}
|
||||||
|
+ onClick={this.onSend}
|
||||||
|
title={_t("composer|send_video_message")}
|
||||||
|
>
|
||||||
|
<SendSolidIcon />
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
diff --git a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
index 851c64e..4285c72 100644
|
||||||
|
--- a/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
+++ b/apps/web/src/components/views/rooms/VideoRecordComposerTile.tsx
|
||||||
|
@@ -30,6 +30,7 @@ interface IState {
|
||||||
|
}
|
||||||
|
|
||||||
|
const PREFERRED_VIDEO_MIME_TYPES = ["video/webm;codecs=vp8,opus", "video/webm"];
|
||||||
|
+const MAX_VIDEO_RECORDING_MS = 120000;
|
||||||
|
|
||||||
|
function preferredVideoMimeType(): string | undefined {
|
||||||
|
for (const mimeType of PREFERRED_VIDEO_MIME_TYPES) {
|
||||||
|
@@ -39,6 +40,18 @@ function preferredVideoMimeType(): string | undefined {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+function isPermissionDeniedError(error: unknown): boolean {
|
||||||
|
+ if (error instanceof DOMException) {
|
||||||
|
+ return error.name === "NotAllowedError" || error.name === "PermissionDeniedError";
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if (error instanceof Error) {
|
||||||
|
+ return /permission denied|not allowed|denied permission/i.test(error.message);
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return false;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
export function isVideoMessageRecorderSupported(): boolean {
|
||||||
|
return (
|
||||||
|
typeof window !== "undefined" &&
|
||||||
|
@@ -56,6 +69,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
private mediaRecorder?: MediaRecorder;
|
||||||
|
private recordedFile?: File;
|
||||||
|
private stream?: MediaStream;
|
||||||
|
+ private durationCapTimer?: number;
|
||||||
|
private isRequestingMedia = false;
|
||||||
|
private isUnmounted = false;
|
||||||
|
private shouldDiscardRecording = false;
|
||||||
|
@@ -72,6 +86,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
public componentWillUnmount(): void {
|
||||||
|
this.isUnmounted = true;
|
||||||
|
this.disposeRecording();
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public onRecordStartEndClick = async (): Promise<void> => {
|
||||||
|
@@ -96,6 +111,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
|
||||||
|
this.isRequestingMedia = true;
|
||||||
|
this.shouldDiscardRecording = false;
|
||||||
|
+ this.clearDurationCapTimer();
|
||||||
|
this.revokePreviewUrl();
|
||||||
|
this.recordedFile = undefined;
|
||||||
|
this.chunks = [];
|
||||||
|
@@ -108,6 +124,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
}
|
||||||
|
|
||||||
|
this.stream = stream;
|
||||||
|
+ this.bindStreamEndedHandlers(stream);
|
||||||
|
const mimeType = preferredVideoMimeType();
|
||||||
|
const mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
|
||||||
|
|
||||||
|
@@ -118,12 +135,13 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
};
|
||||||
|
mediaRecorder.onerror = (ev: Event): void => {
|
||||||
|
logger.error("Error recording video message:", ev);
|
||||||
|
- this.setState({ error: _t("composer|video_message_error") });
|
||||||
|
+ this.failRecording(_t("composer|video_message_error"));
|
||||||
|
};
|
||||||
|
mediaRecorder.onstop = this.onRecorderStop;
|
||||||
|
|
||||||
|
this.mediaRecorder = mediaRecorder;
|
||||||
|
mediaRecorder.start();
|
||||||
|
+ this.durationCapTimer = window.setTimeout(this.onDurationCap, MAX_VIDEO_RECORDING_MS);
|
||||||
|
|
||||||
|
this.setState({ error: undefined, isRecording: true, isSending: false, previewUrl: undefined });
|
||||||
|
this.props.onRecordingStateChange(true);
|
||||||
|
@@ -132,7 +150,9 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
this.stopStream();
|
||||||
|
if (!this.isUnmounted) {
|
||||||
|
this.setState({
|
||||||
|
- error: _t("composer|video_message_error"),
|
||||||
|
+ error: isPermissionDeniedError(e)
|
||||||
|
+ ? _t("composer|video_message_permission_denied")
|
||||||
|
+ : _t("composer|video_message_error"),
|
||||||
|
isRecording: false,
|
||||||
|
isSending: false,
|
||||||
|
previewUrl: undefined,
|
||||||
|
@@ -144,6 +164,21 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
+ private onDurationCap = (): void => {
|
||||||
|
+ this.durationCapTimer = undefined;
|
||||||
|
+
|
||||||
|
+ if (!this.mediaRecorder || this.mediaRecorder.state === "inactive") return;
|
||||||
|
+
|
||||||
|
+ this.stopRecording();
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
+ private onStreamTrackEnded = (): void => {
|
||||||
|
+ if (!this.mediaRecorder || this.mediaRecorder.state === "inactive") return;
|
||||||
|
+
|
||||||
|
+ logger.warn("Video message media stream ended before recording stopped");
|
||||||
|
+ this.failRecording(_t("composer|video_message_error"));
|
||||||
|
+ };
|
||||||
|
+
|
||||||
|
private stopRecording = (): void => {
|
||||||
|
if (!this.mediaRecorder) return;
|
||||||
|
|
||||||
|
@@ -160,6 +195,7 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
const hasRecording = this.chunks.length > 0 && !shouldDiscardRecording;
|
||||||
|
|
||||||
|
this.mediaRecorder = undefined;
|
||||||
|
+ this.clearDurationCapTimer();
|
||||||
|
this.stopStream();
|
||||||
|
|
||||||
|
if (this.isUnmounted) {
|
||||||
|
@@ -200,7 +236,9 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
this.props.replyToEvent,
|
||||||
|
);
|
||||||
|
this.clearRecording();
|
||||||
|
- this.props.onRecordingStateChange(false);
|
||||||
|
+ if (!this.isUnmounted) {
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ }
|
||||||
|
} catch (e) {
|
||||||
|
logger.error("Error sending video message recording:", e);
|
||||||
|
if (!this.isUnmounted) {
|
||||||
|
@@ -219,13 +257,18 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
private clearRecording(): void {
|
||||||
|
this.chunks = [];
|
||||||
|
this.recordedFile = undefined;
|
||||||
|
+ this.clearDurationCapTimer();
|
||||||
|
+ this.stopStream();
|
||||||
|
this.revokePreviewUrl();
|
||||||
|
+ if (this.isUnmounted) return;
|
||||||
|
+
|
||||||
|
this.setState({ error: undefined, isRecording: false, isSending: false, previewUrl: undefined });
|
||||||
|
}
|
||||||
|
|
||||||
|
private disposeRecording(): void {
|
||||||
|
this.chunks = [];
|
||||||
|
this.recordedFile = undefined;
|
||||||
|
+ this.clearDurationCapTimer();
|
||||||
|
this.stopStream();
|
||||||
|
this.revokePreviewUrl();
|
||||||
|
|
||||||
|
@@ -241,8 +284,32 @@ export default class VideoRecordComposerTile extends React.PureComponent<IProps,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ private failRecording(error: string): void {
|
||||||
|
+ this.shouldDiscardRecording = true;
|
||||||
|
+ this.disposeRecording();
|
||||||
|
+
|
||||||
|
+ if (this.isUnmounted) return;
|
||||||
|
+
|
||||||
|
+ this.setState({ error, isRecording: false, isSending: false, previewUrl: undefined });
|
||||||
|
+ this.props.onRecordingStateChange(false);
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ private clearDurationCapTimer(): void {
|
||||||
|
+ if (this.durationCapTimer !== undefined) {
|
||||||
|
+ window.clearTimeout(this.durationCapTimer);
|
||||||
|
+ this.durationCapTimer = undefined;
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ private bindStreamEndedHandlers(stream: MediaStream): void {
|
||||||
|
+ stream.getTracks().forEach((track) => track.addEventListener("ended", this.onStreamTrackEnded));
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
private stopStream(): void {
|
||||||
|
- this.stream?.getTracks().forEach((track) => track.stop());
|
||||||
|
+ this.stream?.getTracks().forEach((track) => {
|
||||||
|
+ track.removeEventListener("ended", this.onStreamTrackEnded);
|
||||||
|
+ track.stop();
|
||||||
|
+ });
|
||||||
|
this.stream = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json
|
||||||
|
index e11fe90..83b7c21 100644
|
||||||
|
--- a/apps/web/src/i18n/strings/en_EN.json
|
||||||
|
+++ b/apps/web/src/i18n/strings/en_EN.json
|
||||||
|
@@ -648,6 +648,7 @@
|
||||||
|
"video_message_button": "Record video message",
|
||||||
|
"video_message_cancel": "Cancel video message",
|
||||||
|
"video_message_error": "Unable to record video message",
|
||||||
|
+ "video_message_permission_denied": "Camera permission denied",
|
||||||
|
"video_message_preview": "Video message preview",
|
||||||
|
"voice_message_button": "Voice Message"
|
||||||
|
},
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# Element Web patches
|
||||||
|
|
||||||
|
Put ordered local patch files in this directory as `*.patch`.
|
||||||
|
|
||||||
|
- Files are applied in lexical order.
|
||||||
|
- Use zero-padded numeric prefixes when patch order matters, e.g. `0001-...patch`.
|
||||||
|
- Keep non-patch files out of the order by using a different extension; `README.md` is ignored by the Nix filter.
|
||||||
|
|
||||||
|
## Regenerating a patch
|
||||||
|
|
||||||
|
Create a clean checkout of upstream Element Web, make the change under `apps/web`, then run from the repository root:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git diff -- apps/web > package/element-web/patches/0001-description.patch
|
||||||
|
```
|
||||||
|
|
||||||
|
Use one patch per logical change so the sequence stays reproducible and reviewable.
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
{
|
||||||
|
bash,
|
||||||
|
cacert,
|
||||||
|
coreutils,
|
||||||
|
dockerTools,
|
||||||
|
git,
|
||||||
|
lib,
|
||||||
|
nix,
|
||||||
|
symlinkJoin,
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
name = "gitea-runner-nix-image";
|
||||||
|
tag = "2026-06-07";
|
||||||
|
root = symlinkJoin {
|
||||||
|
name = "${name}-root";
|
||||||
|
paths = [
|
||||||
|
bash
|
||||||
|
cacert
|
||||||
|
coreutils
|
||||||
|
git
|
||||||
|
nix
|
||||||
|
];
|
||||||
|
};
|
||||||
|
in
|
||||||
|
dockerTools.buildLayeredImageWithNixDb {
|
||||||
|
inherit name tag;
|
||||||
|
|
||||||
|
contents = [ root ];
|
||||||
|
|
||||||
|
fakeRootCommands = ''
|
||||||
|
mkdir -p ./etc
|
||||||
|
cat > ./etc/passwd <<'EOF'
|
||||||
|
root:x:0:0:root:/root:/bin/bash
|
||||||
|
nobody:x:65534:65534:nobody:/var/empty:/bin/false
|
||||||
|
EOF
|
||||||
|
cat > ./etc/group <<'EOF'
|
||||||
|
root:x:0:
|
||||||
|
nixbld:x:30000:
|
||||||
|
nobody:x:65534:
|
||||||
|
EOF
|
||||||
|
for n in $(seq 1 10); do
|
||||||
|
echo "nixbld$n:x:$((30000 + n)):30000:Nix build user $n:/var/empty:/bin/false" >> ./etc/passwd
|
||||||
|
sed -i "s/^nixbld:x:30000:.*/&,nixbld$n/" ./etc/group
|
||||||
|
done
|
||||||
|
|
||||||
|
mkdir -p ./nix/var/nix/{gcroots,profiles,temproots,userpool,db}
|
||||||
|
mkdir -p ./nix/var/log/nix/drvs
|
||||||
|
chmod 1777 ./nix/var/nix/gcroots ./nix/var/nix/profiles
|
||||||
|
'';
|
||||||
|
|
||||||
|
extraCommands = ''
|
||||||
|
mkdir -p etc/nix root tmp
|
||||||
|
chmod 1777 tmp
|
||||||
|
|
||||||
|
cat > etc/nix/nix.conf <<'EOF'
|
||||||
|
accept-flake-config = true
|
||||||
|
experimental-features = nix-command flakes
|
||||||
|
substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
|
||||||
|
trusted-users = root
|
||||||
|
sandbox = false
|
||||||
|
EOF
|
||||||
|
'';
|
||||||
|
|
||||||
|
config = {
|
||||||
|
Cmd = [ "${bash}/bin/bash" ];
|
||||||
|
Env = [
|
||||||
|
"HOME=/root"
|
||||||
|
"PATH=${lib.makeBinPath [ bash coreutils git nix ]}"
|
||||||
|
"SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||||
|
"NIX_SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||||
|
"USER=root"
|
||||||
|
];
|
||||||
|
Labels = {
|
||||||
|
"org.opencontainers.image.description" = "Nix-capable Gitea Actions job image";
|
||||||
|
"org.opencontainers.image.ref.name" = "${name}:${tag}";
|
||||||
|
"org.opencontainers.image.source" = "https://gitea.hectic-lab.com/hectic-lab/util.nix";
|
||||||
|
};
|
||||||
|
WorkingDir = "/workspace";
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
|
|||||||
Commands: []*cli.Command{
|
Commands: []*cli.Command{
|
||||||
newUserCommand(),
|
newUserCommand(),
|
||||||
newRepoSyncReleasesCommand(),
|
newRepoSyncReleasesCommand(),
|
||||||
|
newHeatmapCommand(),
|
||||||
newRegenerateCommand(),
|
newRegenerateCommand(),
|
||||||
newAuthCommand(),
|
newAuthCommand(),
|
||||||
newSendMailCommand(),
|
newSendMailCommand(),
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
repo_service "code.gitea.io/gitea/services/repository"
|
||||||
|
|
||||||
|
"github.com/urfave/cli/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newHeatmapCommand() *cli.Command {
|
||||||
|
return &cli.Command{
|
||||||
|
Name: "heatmap",
|
||||||
|
Usage: "Manage heatmap indexes",
|
||||||
|
Commands: []*cli.Command{
|
||||||
|
newHeatmapReindexCommand(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHeatmapReindexCommand() *cli.Command {
|
||||||
|
return &cli.Command{
|
||||||
|
Name: "reindex",
|
||||||
|
Usage: "Reindex heatmap contributions from repository default branches",
|
||||||
|
Action: runHeatmapReindex,
|
||||||
|
Flags: []cli.Flag{
|
||||||
|
&cli.BoolFlag{
|
||||||
|
Name: "all",
|
||||||
|
Usage: "Reindex all repositories",
|
||||||
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
Name: "repo",
|
||||||
|
Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
|
||||||
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
Name: "owner",
|
||||||
|
Usage: "Owner name for --repo",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
|
||||||
|
if err := initDB(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := git.InitSimple(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
reindexAll := c.Bool("all")
|
||||||
|
repoName := c.String("repo")
|
||||||
|
ownerName := c.String("owner")
|
||||||
|
if reindexAll {
|
||||||
|
if repoName != "" || ownerName != "" {
|
||||||
|
return fmt.Errorf("--all cannot be combined with --repo or --owner")
|
||||||
|
}
|
||||||
|
return reindexAllHeatmapRepositories(ctx)
|
||||||
|
}
|
||||||
|
if repoName == "" {
|
||||||
|
return fmt.Errorf("either --all or --repo must be provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return reindexHeatmapRepository(ctx, repo)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
|
||||||
|
if ownerName != "" {
|
||||||
|
if strings.Contains(repoName, "/") {
|
||||||
|
return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
|
||||||
|
}
|
||||||
|
return ownerName, repoName, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ownerName, repoName, ok := strings.Cut(repoName, "/")
|
||||||
|
if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
|
||||||
|
return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
|
||||||
|
}
|
||||||
|
return ownerName, repoName, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reindexAllHeatmapRepositories(ctx context.Context) error {
|
||||||
|
for page := 1; ; page++ {
|
||||||
|
repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
|
||||||
|
ListOptions: db.ListOptions{
|
||||||
|
PageSize: repo_model.RepositoryListDefaultPageSize,
|
||||||
|
Page: page,
|
||||||
|
},
|
||||||
|
Private: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("SearchRepositoryByName: %w", err)
|
||||||
|
}
|
||||||
|
if len(repos) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
log.Trace("Processing next %d repos of %d", len(repos), count)
|
||||||
|
for _, repo := range repos {
|
||||||
|
if err := reindexHeatmapRepository(ctx, repo); err != nil {
|
||||||
|
log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
|
||||||
|
log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
|
||||||
|
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
repo_service "code.gitea.io/gitea/services/repository"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeatmapAdminReindex(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
|
||||||
|
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||||
|
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||||
|
contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
|
||||||
|
|
||||||
|
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
|
||||||
|
newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
|
||||||
|
{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
})
|
||||||
|
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
|
||||||
|
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
|
||||||
|
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
|
||||||
|
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeatmapAdminRepoSelector(t *testing.T) {
|
||||||
|
owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "user", owner)
|
||||||
|
assert.Equal(t, "repo", repo)
|
||||||
|
|
||||||
|
owner, repo, err = parseHeatmapRepoSelector("user", "repo")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "user", owner)
|
||||||
|
assert.Equal(t, "repo", repo)
|
||||||
|
|
||||||
|
_, _, err = parseHeatmapRepoSelector("", "repo")
|
||||||
|
assert.ErrorContains(t, err, "owner/name")
|
||||||
|
_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
|
||||||
|
assert.ErrorContains(t, err, "when --owner is set")
|
||||||
|
}
|
||||||
|
|
||||||
|
type heatmapAdminTestCommit struct {
|
||||||
|
Branch string
|
||||||
|
Mark string
|
||||||
|
Parent string
|
||||||
|
AuthorName string
|
||||||
|
AuthorEmail string
|
||||||
|
CommitterName string
|
||||||
|
CommitterEmail string
|
||||||
|
AuthorDate string
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
|
||||||
|
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
|
repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
marks := runHeatmapAdminFastImport(t, repo, commits)
|
||||||
|
repo.IsEmpty = false
|
||||||
|
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
|
||||||
|
return repo, marks
|
||||||
|
}
|
||||||
|
|
||||||
|
func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var stream strings.Builder
|
||||||
|
branchTips := make(map[string]string)
|
||||||
|
for i, commit := range commits {
|
||||||
|
mark := fmt.Sprintf(":%d", i+1)
|
||||||
|
branchRef := "refs/heads/" + commit.Branch
|
||||||
|
stream.WriteString("commit " + branchRef + "\n")
|
||||||
|
stream.WriteString("mark " + mark + "\n")
|
||||||
|
stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
|
||||||
|
stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
|
||||||
|
message := "heatmap admin " + commit.Mark
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
|
||||||
|
if parentMark := branchTips[commit.Branch]; parentMark != "" {
|
||||||
|
stream.WriteString("from " + parentMark + "\n")
|
||||||
|
} else if commit.Parent != "" {
|
||||||
|
stream.WriteString("from " + commit.Parent + "\n")
|
||||||
|
}
|
||||||
|
content := commit.Mark + "\n"
|
||||||
|
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
|
||||||
|
branchTips[commit.Branch] = mark
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
WithStdinCopy(strings.NewReader(stream.String())).
|
||||||
|
Run(t.Context()))
|
||||||
|
|
||||||
|
commitSHAs := make(map[string]string, len(commits))
|
||||||
|
for _, commit := range commits {
|
||||||
|
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
|
||||||
|
AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
|
||||||
|
AddDynamicArguments("refs/heads/" + commit.Branch).
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
RunStdString(t.Context())
|
||||||
|
require.NoError(t, err)
|
||||||
|
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
|
||||||
|
}
|
||||||
|
return commitSHAs
|
||||||
|
}
|
||||||
|
|
||||||
|
func heatmapAdminSignatureLine(kind, name, email, date string) string {
|
||||||
|
parsed, err := time.Parse(time.RFC3339, date)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
|
||||||
|
t.Helper()
|
||||||
|
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return contributions
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package activities
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"xorm.io/xorm/schemas"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HeatmapContribution stores commit contributions counted for profile heatmaps.
|
||||||
|
type HeatmapContribution struct {
|
||||||
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
UserID int64 `xorm:"NOT NULL"`
|
||||||
|
RepoID int64 `xorm:"NOT NULL"`
|
||||||
|
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
|
||||||
|
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
|
||||||
|
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
|
||||||
|
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||||
|
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing.
|
||||||
|
type HeatmapContributionIdentity struct {
|
||||||
|
RepoID int64
|
||||||
|
CommitSHA string
|
||||||
|
UserID int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
db.RegisterModel(new(HeatmapContribution))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TableIndices implements xorm's TableIndices interface.
|
||||||
|
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
|
||||||
|
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
|
||||||
|
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
|
||||||
|
|
||||||
|
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
|
||||||
|
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
|
||||||
|
|
||||||
|
return []*schemas.Index{uniqueContribution, userAuthorRepo}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertHeatmapContribution creates or updates a commit contribution without duplicating counted rows.
|
||||||
|
func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContribution) error {
|
||||||
|
return db.WithTx(ctx, func(ctx context.Context) error {
|
||||||
|
e := db.GetEngine(ctx)
|
||||||
|
|
||||||
|
rows, err := e.Where("repo_id=? AND commit_sha=? AND user_id=?", contribution.RepoID, contribution.CommitSHA, contribution.UserID).
|
||||||
|
Cols("author_email", "author_unix").
|
||||||
|
Update(contribution)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if rows > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
has, err := e.Exist(&HeatmapContribution{RepoID: contribution.RepoID, CommitSHA: contribution.CommitSHA, UserID: contribution.UserID})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if has {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = e.Insert(contribution)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountHeatmapContributions returns counted heatmap contribution rows for a user.
|
||||||
|
func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) {
|
||||||
|
return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution))
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities.
|
||||||
|
func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error {
|
||||||
|
if len(identities) == 0 {
|
||||||
|
_, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
current := make(map[HeatmapContributionIdentity]struct{}, len(identities))
|
||||||
|
for _, identity := range identities {
|
||||||
|
current[identity] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
contributions, err := FindHeatmapContributionsByRepo(ctx, repoID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, contribution := range contributions {
|
||||||
|
identity := HeatmapContributionIdentity{
|
||||||
|
RepoID: contribution.RepoID,
|
||||||
|
CommitSHA: contribution.CommitSHA,
|
||||||
|
UserID: contribution.UserID,
|
||||||
|
}
|
||||||
|
if _, ok := current[identity]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository.
|
||||||
|
func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) {
|
||||||
|
contributions := make([]*HeatmapContribution, 0)
|
||||||
|
return contributions, db.GetEngine(ctx).
|
||||||
|
Where("repo_id=?", repoID).
|
||||||
|
OrderBy("author_unix ASC, commit_sha ASC, user_id ASC").
|
||||||
|
Find(&contributions)
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package activities_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeatmapContributionModel(t *testing.T) {
|
||||||
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
assert.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
|
||||||
|
const (
|
||||||
|
userID = int64(2)
|
||||||
|
repoID = int64(1)
|
||||||
|
commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
authorUnix = timeutil.TimeStamp(1579089600)
|
||||||
|
)
|
||||||
|
|
||||||
|
contribution := &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: "user2@example.com",
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), contribution))
|
||||||
|
|
||||||
|
count, err := activities_model.CountHeatmapContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.EqualValues(t, 1, count)
|
||||||
|
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: "changed@example.com",
|
||||||
|
AuthorUnix: authorUnix + 900,
|
||||||
|
}))
|
||||||
|
|
||||||
|
count, err = activities_model.CountHeatmapContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.EqualValues(t, 1, count)
|
||||||
|
|
||||||
|
stored := &activities_model.HeatmapContribution{UserID: userID, RepoID: repoID, CommitSHA: commitSHA}
|
||||||
|
has, err := db.GetEngine(t.Context()).Get(stored)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.True(t, has)
|
||||||
|
assert.Equal(t, "changed@example.com", stored.AuthorEmail)
|
||||||
|
assert.Equal(t, authorUnix+900, stored.AuthorUnix)
|
||||||
|
assert.NotZero(t, stored.CreatedUnix)
|
||||||
|
assert.NotZero(t, stored.UpdatedUnix)
|
||||||
|
|
||||||
|
err = db.Insert(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: "duplicate@example.com",
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
})
|
||||||
|
assert.Error(t, err)
|
||||||
|
|
||||||
|
includePrivate, err := user_model.GetIncludePrivateContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, includePrivate)
|
||||||
|
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), userID, true))
|
||||||
|
includePrivate, err = user_model.GetIncludePrivateContributions(t.Context(), userID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, includePrivate)
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"code.gitea.io/gitea/models/organization"
|
"code.gitea.io/gitea/models/organization"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -38,35 +39,45 @@ func getUserHeatmapData(ctx context.Context, user *user_model.User, team *organi
|
|||||||
|
|
||||||
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
|
// Group by 15 minute intervals which will allow the client to accurately shift the timestamp to their timezone.
|
||||||
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
|
// The interval is based on the fact that there are timezones such as UTC +5:30 and UTC +12:45.
|
||||||
groupBy := "created_unix / 900 * 900"
|
groupBy := "author_unix / 900 * 900"
|
||||||
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
|
groupByName := "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
|
||||||
switch {
|
switch {
|
||||||
case setting.Database.Type.IsMySQL():
|
case setting.Database.Type.IsMySQL():
|
||||||
groupBy = "created_unix DIV 900 * 900"
|
groupBy = "author_unix DIV 900 * 900"
|
||||||
case setting.Database.Type.IsMSSQL():
|
case setting.Database.Type.IsMSSQL():
|
||||||
groupByName = groupBy
|
groupByName = groupBy
|
||||||
}
|
}
|
||||||
|
|
||||||
cond, err := ActivityQueryCondition(ctx, GetFeedsOptions{
|
includePrivate, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
|
||||||
RequestedUser: user,
|
|
||||||
RequestedTeam: team,
|
|
||||||
Actor: doer,
|
|
||||||
IncludePrivate: true, // don't filter by private, as we already filter by repo access
|
|
||||||
IncludeDeleted: true,
|
|
||||||
// * Heatmaps for individual users only include actions that the user themself did.
|
|
||||||
// * For organizations actions by all users that were made in owned
|
|
||||||
// repositories are counted.
|
|
||||||
OnlyPerformedBy: !user.IsOrganization(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return hdata, db.GetEngine(ctx).
|
sess := db.GetEngine(ctx).
|
||||||
Select(groupBy+" AS timestamp, count(user_id) as contributions").
|
Select(groupBy+" AS timestamp, count(heatmap_contribution.user_id) as contributions").
|
||||||
Table("action").
|
Table("heatmap_contribution").
|
||||||
Where(cond).
|
Join("INNER", "repository", "repository.id = heatmap_contribution.repo_id").
|
||||||
And("created_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
|
Join("INNER", "`user` AS repo_owner", "repo_owner.id = repository.owner_id").
|
||||||
|
Where("author_unix > ?", timeutil.TimeStampNow()-(366+7)*86400). // (366+7) days to include the first week for the heatmap
|
||||||
|
And("author_unix <= ?", timeutil.TimeStampNow())
|
||||||
|
|
||||||
|
if !includePrivate {
|
||||||
|
sess = sess.And("repository.is_private = ?", false).
|
||||||
|
And("repo_owner.visibility = ?", structs.VisibleTypePublic)
|
||||||
|
}
|
||||||
|
|
||||||
|
if user.IsOrganization() {
|
||||||
|
sess = sess.And("repository.owner_id = ?", user.ID)
|
||||||
|
if team != nil && !team.IncludesAllRepositories {
|
||||||
|
sess = sess.Join("INNER", "team_repo", "team_repo.repo_id = repository.id").
|
||||||
|
And("team_repo.org_id = ?", team.OrgID).
|
||||||
|
And("team_repo.team_id = ?", team.ID)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sess = sess.And("heatmap_contribution.user_id = ?", user.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hdata, sess.
|
||||||
GroupBy(groupByName).
|
GroupBy(groupByName).
|
||||||
OrderBy("timestamp").
|
OrderBy("timestamp").
|
||||||
Find(&hdata)
|
Find(&hdata)
|
||||||
|
|||||||
@@ -4,19 +4,43 @@
|
|||||||
package activities_test
|
package activities_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha1"
|
||||||
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
activities_model "code.gitea.io/gitea/models/activities"
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/models/organization"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
"code.gitea.io/gitea/models/unittest"
|
"code.gitea.io/gitea/models/unittest"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/json"
|
"code.gitea.io/gitea/modules/json"
|
||||||
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGetUserHeatmapDataByUser(t *testing.T) {
|
func TestGetUserHeatmapDataByUser(t *testing.T) {
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
|
||||||
|
// Mock time
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
insertHeatmapContribution(t, 2, 1, "user2-public-author-date", 1603009800)
|
||||||
|
insertHeatmapContribution(t, 2, 1, "user2-public-same-bucket", 1603009850)
|
||||||
|
insertHeatmapContribution(t, 2, 2, "user2-private-hidden", 1603010700)
|
||||||
|
insertHeatmapContribution(t, 2, 4, "user2-hidden-owner-hidden", 1603010700)
|
||||||
|
insertHeatmapContribution(t, 2, 1, "user2-future-hidden", timeutil.TimeStampNow()+900)
|
||||||
|
insertHeatmapContribution(t, 2, 999999, "user2-deleted-repo-hidden", 1603011600)
|
||||||
|
insertHeatmapContribution(t, 3, 1, "other-author-ignored", 1603010700)
|
||||||
|
setUserVisibility(t, 5, structs.VisibleTypePrivate)
|
||||||
|
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
desc string
|
desc string
|
||||||
userID int64
|
userID int64
|
||||||
@@ -25,73 +49,211 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
|
|||||||
JSONResult string
|
JSONResult string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"self looks at action in private repo",
|
"self sees public author-date contributions only",
|
||||||
2, 2, 1, `[{"timestamp":1603227600,"contributions":1}]`,
|
2, 2, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"admin looks at action in private repo",
|
"admin sees public author-date contributions only",
|
||||||
2, 1, 1, `[{"timestamp":1603227600,"contributions":1}]`,
|
2, 1, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"other user looks at action in private repo",
|
"other user sees public author-date contributions only",
|
||||||
2, 3, 0, `[]`,
|
2, 3, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"nobody looks at action in private repo",
|
"anonymous sees public author-date contributions only",
|
||||||
2, 0, 0, `[]`,
|
2, 0, 2, `[{"timestamp":1603009800,"contributions":2}]`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"collaborator looks at action in private repo",
|
"different author is not counted for target user",
|
||||||
16, 15, 1, `[{"timestamp":1603267200,"contributions":1}]`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no action action not performed by target user",
|
|
||||||
3, 3, 0, `[]`,
|
3, 3, 0, `[]`,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"multiple actions performed with two grouped together",
|
|
||||||
10, 10, 3, `[{"timestamp":1603009800,"contributions":1},{"timestamp":1603010700,"contributions":2}]`,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
// Prepare
|
|
||||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
|
||||||
|
|
||||||
// Mock time
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
|
||||||
|
|
||||||
|
var doer *user_model.User
|
||||||
|
if tc.doerID != 0 {
|
||||||
|
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
|
||||||
|
}
|
||||||
|
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.CountResult, countHeatmapContributions(heatmap), "testcase '%s'", tc.desc)
|
||||||
|
|
||||||
|
// Test JSON rendering
|
||||||
|
jsonData, err := json.Marshal(heatmap)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.JSONEq(t, tc.JSONResult, string(jsonData))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetUserHeatmapDataByUserUsesCurrentRepoVisibility(t *testing.T) {
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
|
||||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
defer timeutil.MockUnset()
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
for _, tc := range testCases {
|
insertHeatmapContribution(t, 2, 1, "user2-visibility-flip", 1603009800)
|
||||||
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.userID})
|
|
||||||
|
|
||||||
var doer *user_model.User
|
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
if tc.doerID != 0 {
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: tc.doerID})
|
require.NoError(t, err)
|
||||||
}
|
assertHeatmapJSON(t, heatmap, `[{
|
||||||
|
"timestamp": 1603009800,
|
||||||
|
"contributions": 1
|
||||||
|
}]`)
|
||||||
|
|
||||||
// get the action for comparison
|
setRepoPrivate(t, 1, true)
|
||||||
actions, count, err := activities_model.GetFeeds(t.Context(), activities_model.GetFeedsOptions{
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
RequestedUser: user,
|
require.NoError(t, err)
|
||||||
Actor: doer,
|
assert.Empty(t, heatmap)
|
||||||
IncludePrivate: true,
|
|
||||||
OnlyPerformedBy: true,
|
|
||||||
IncludeDeleted: true,
|
|
||||||
})
|
|
||||||
assert.NoError(t, err)
|
|
||||||
|
|
||||||
// Get the heatmap and compare
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, true))
|
||||||
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), user, doer)
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
var contributions int
|
require.NoError(t, err)
|
||||||
for _, hm := range heatmap {
|
assertHeatmapJSON(t, heatmap, `[{
|
||||||
contributions += int(hm.Contributions)
|
"timestamp": 1603009800,
|
||||||
}
|
"contributions": 1
|
||||||
assert.NoError(t, err)
|
}]`)
|
||||||
assert.Len(t, actions, contributions, "invalid action count: did the test data became too old?")
|
}
|
||||||
assert.Equal(t, count, int64(contributions))
|
|
||||||
assert.Equal(t, tc.CountResult, contributions, "testcase '%s'", tc.desc)
|
|
||||||
|
|
||||||
// Test JSON rendering
|
func TestGetUserHeatmapDataByOrgTeam(t *testing.T) {
|
||||||
jsonData, err := json.Marshal(heatmap)
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
assert.NoError(t, err)
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
assert.JSONEq(t, tc.JSONResult, string(jsonData))
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 3, false))
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
insertHeatmapContribution(t, 2, 32, "org-team-public", 1603009800)
|
||||||
|
insertHeatmapContribution(t, 2, 3, "org-team-private-hidden", 1603010700)
|
||||||
|
insertHeatmapContribution(t, 2, 1, "non-org-repo-ignored", 1603011600)
|
||||||
|
|
||||||
|
org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3})
|
||||||
|
team := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: 7})
|
||||||
|
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
|
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByOrgTeam(t.Context(), org, team, doer)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, countHeatmapContributions(heatmap))
|
||||||
|
|
||||||
|
jsonData, err := json.Marshal(heatmap)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.JSONEq(t, `[{"timestamp":1603009800,"contributions":1}]`, string(jsonData))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUserHeatmapPrivateContributionsOptIn(t *testing.T) {
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, false))
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
insertHeatmapContribution(t, 16, 21, "user16-public", 1603009800)
|
||||||
|
insertHeatmapContribution(t, 16, 22, "user16-private-one", 1603009850)
|
||||||
|
insertHeatmapContribution(t, 16, 22, "user16-private-two", 1603010700)
|
||||||
|
|
||||||
|
target := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 16})
|
||||||
|
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||||
|
authenticated := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3})
|
||||||
|
collaborator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15})
|
||||||
|
|
||||||
|
for _, doer := range []*user_model.User{nil, target, admin, authenticated, collaborator} {
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, doer)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, countHeatmapContributions(heatmap), "private contributions should be hidden by default from %s", heatmapDoerName(doer))
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 16, true))
|
||||||
|
heatmap, err := activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assertHeatmapJSON(t, heatmap, `[{"timestamp":1603009800,"contributions":2},{"timestamp":1603010700,"contributions":1}]`)
|
||||||
|
|
||||||
|
target.KeepActivityPrivate = true
|
||||||
|
_, err = db.GetEngine(t.Context()).ID(target.ID).Cols("keep_activity_private").Update(target)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, heatmap)
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, authenticated)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, heatmap)
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, target)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 3, countHeatmapContributions(heatmap))
|
||||||
|
|
||||||
|
heatmap, err = activities_model.GetUserHeatmapDataByUser(t.Context(), target, admin)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 3, countHeatmapContributions(heatmap))
|
||||||
|
}
|
||||||
|
|
||||||
|
func insertHeatmapContribution(t *testing.T, userID, repoID int64, commitSHA string, authorUnix timeutil.TimeStamp) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: fmt.Sprintf("%040x", sha1.Sum([]byte(commitSHA))),
|
||||||
|
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func setRepoPrivate(t *testing.T, repoID int64, isPrivate bool) {
|
||||||
|
t.Helper()
|
||||||
|
repo := &repo_model.Repository{ID: repoID, IsPrivate: isPrivate}
|
||||||
|
_, err := db.GetEngine(t.Context()).ID(repoID).Cols("is_private").Update(repo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setUserVisibility(t *testing.T, userID int64, visibility structs.VisibleType) {
|
||||||
|
t.Helper()
|
||||||
|
user := &user_model.User{ID: userID, Visibility: visibility}
|
||||||
|
_, err := db.GetEngine(t.Context()).ID(userID).Cols("visibility").Update(user)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertHeatmapJSON(t *testing.T, heatmap []*activities_model.UserHeatmapData, expected string) {
|
||||||
|
t.Helper()
|
||||||
|
jsonData, err := json.Marshal(heatmap)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.JSONEq(t, expected, string(jsonData))
|
||||||
|
|
||||||
|
var decoded []map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(jsonData, &decoded))
|
||||||
|
for _, entry := range decoded {
|
||||||
|
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, mapKeys(entry))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mapKeys[K comparable, V any](m map[K]V) []K {
|
||||||
|
keys := make([]K, 0, len(m))
|
||||||
|
for key := range m {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
func countHeatmapContributions(heatmap []*activities_model.UserHeatmapData) int {
|
||||||
|
var contributions int
|
||||||
|
for _, hm := range heatmap {
|
||||||
|
contributions += int(hm.Contributions)
|
||||||
|
}
|
||||||
|
return contributions
|
||||||
|
}
|
||||||
|
|
||||||
|
func heatmapDoerName(doer *user_model.User) string {
|
||||||
|
if doer == nil {
|
||||||
|
return "anonymous"
|
||||||
|
}
|
||||||
|
return doer.Name
|
||||||
|
}
|
||||||
|
|||||||
@@ -405,6 +405,7 @@ func prepareMigrationTasks() []*migration {
|
|||||||
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
|
newMigration(328, "Add TokenPermissions column to ActionRunJob", v1_26.AddTokenPermissionsToActionRunJob),
|
||||||
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
|
newMigration(329, "Add unique constraint for user badge", v1_26.AddUniqueIndexForUserBadge),
|
||||||
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
|
newMigration(330, "Add name column to webhook", v1_26.AddNameToWebhook),
|
||||||
|
newMigration(331, "Create heatmap contribution table", v1_26.CreateHeatmapContributionTable),
|
||||||
}
|
}
|
||||||
return preparedMigrations
|
return preparedMigrations
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package v1_26
|
||||||
|
|
||||||
|
import (
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"xorm.io/xorm"
|
||||||
|
"xorm.io/xorm/schemas"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HeatmapContribution struct { //revive:disable-line:exported
|
||||||
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
UserID int64 `xorm:"NOT NULL"`
|
||||||
|
RepoID int64 `xorm:"NOT NULL"`
|
||||||
|
CommitSHA string `xorm:"VARCHAR(64) NOT NULL"`
|
||||||
|
AuthorEmail string `xorm:"VARCHAR(320) NOT NULL"`
|
||||||
|
AuthorUnix timeutil.TimeStamp `xorm:"NOT NULL"`
|
||||||
|
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
||||||
|
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TableIndices implements xorm's TableIndices interface.
|
||||||
|
func (c *HeatmapContribution) TableIndices() []*schemas.Index {
|
||||||
|
uniqueContribution := schemas.NewIndex("repo_commit_user", schemas.UniqueType)
|
||||||
|
uniqueContribution.AddColumn("repo_id", "commit_sha", "user_id")
|
||||||
|
|
||||||
|
userAuthorRepo := schemas.NewIndex("u_a_r", schemas.IndexType)
|
||||||
|
userAuthorRepo.AddColumn("user_id", "author_unix", "repo_id")
|
||||||
|
|
||||||
|
return []*schemas.Index{uniqueContribution, userAuthorRepo}
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateHeatmapContributionTable(x *xorm.Engine) error {
|
||||||
|
_, err := x.SyncWithOptions(xorm.SyncOptions{IgnoreDropIndices: true}, new(HeatmapContribution))
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"code.gitea.io/gitea/models/db"
|
"code.gitea.io/gitea/models/db"
|
||||||
@@ -19,6 +20,8 @@ import (
|
|||||||
"xorm.io/xorm/convert"
|
"xorm.io/xorm/convert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const SettingsKeyIncludePrivateContributions = "include_private_contributions"
|
||||||
|
|
||||||
// Setting is a key value store of user settings
|
// Setting is a key value store of user settings
|
||||||
type Setting struct {
|
type Setting struct {
|
||||||
ID int64 `xorm:"pk autoincr"`
|
ID int64 `xorm:"pk autoincr"`
|
||||||
@@ -254,3 +257,17 @@ func SetUserSettingJSON[T any](ctx context.Context, userID int64, key string, va
|
|||||||
}
|
}
|
||||||
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
|
return SetUserSetting(ctx, userID, key, util.UnsafeBytesToString(bs))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetIncludePrivateContributions returns whether a user opted in to private heatmap contributions.
|
||||||
|
func GetIncludePrivateContributions(ctx context.Context, userID int64) (bool, error) {
|
||||||
|
value, err := GetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, "false")
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return strconv.ParseBool(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetIncludePrivateContributions stores whether a user opted in to private heatmap contributions.
|
||||||
|
func SetIncludePrivateContributions(ctx context.Context, userID int64, include bool) error {
|
||||||
|
return SetUserSetting(ctx, userID, SettingsKeyIncludePrivateContributions, strconv.FormatBool(include))
|
||||||
|
}
|
||||||
|
|||||||
@@ -80,8 +80,9 @@ type UserSettings struct {
|
|||||||
Theme string `json:"theme"`
|
Theme string `json:"theme"`
|
||||||
DiffViewStyle string `json:"diff_view_style"`
|
DiffViewStyle string `json:"diff_view_style"`
|
||||||
// Privacy
|
// Privacy
|
||||||
HideEmail bool `json:"hide_email"`
|
HideEmail bool `json:"hide_email"`
|
||||||
HideActivity bool `json:"hide_activity"`
|
HideActivity bool `json:"hide_activity"`
|
||||||
|
IncludePrivateContributions bool `json:"include_private_contributions"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// UserSettingsOptions represents options to change user settings
|
// UserSettingsOptions represents options to change user settings
|
||||||
@@ -95,8 +96,9 @@ type UserSettingsOptions struct {
|
|||||||
Theme *string `json:"theme"`
|
Theme *string `json:"theme"`
|
||||||
DiffViewStyle *string `json:"diff_view_style"`
|
DiffViewStyle *string `json:"diff_view_style"`
|
||||||
// Privacy
|
// Privacy
|
||||||
HideEmail *bool `json:"hide_email"`
|
HideEmail *bool `json:"hide_email"`
|
||||||
HideActivity *bool `json:"hide_activity"`
|
HideActivity *bool `json:"hide_activity"`
|
||||||
|
IncludePrivateContributions *bool `json:"include_private_contributions"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RenameUserOption options when renaming a user
|
// RenameUserOption options when renaming a user
|
||||||
|
|||||||
@@ -708,6 +708,8 @@
|
|||||||
"settings.privacy": "Privacy",
|
"settings.privacy": "Privacy",
|
||||||
"settings.keep_activity_private": "Hide Activity from profile page",
|
"settings.keep_activity_private": "Hide Activity from profile page",
|
||||||
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
|
"settings.keep_activity_private_popup": "Makes the activity visible only for you and the admins",
|
||||||
|
"settings.include_private_contributions": "Show private contributions on profile heatmap",
|
||||||
|
"settings.include_private_contributions_popup": "Publicly reveals only contribution dates and counts from non-public repositories, never repository or commit details.",
|
||||||
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
|
"settings.lookup_avatar_by_mail": "Look Up Avatar by Email Address",
|
||||||
"settings.federated_avatar_lookup": "Federated Avatar Lookup",
|
"settings.federated_avatar_lookup": "Federated Avatar Lookup",
|
||||||
"settings.enable_custom_avatar": "Use Custom Avatar",
|
"settings.enable_custom_avatar": "Use Custom Avatar",
|
||||||
|
|||||||
@@ -24,7 +24,12 @@ func GetUserSettings(ctx *context.APIContext) {
|
|||||||
// responses:
|
// responses:
|
||||||
// "200":
|
// "200":
|
||||||
// "$ref": "#/responses/UserSettings"
|
// "$ref": "#/responses/UserSettings"
|
||||||
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
|
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
|
||||||
|
if err != nil {
|
||||||
|
ctx.APIErrorInternal(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.JSON(http.StatusOK, settings)
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateUserSettings returns user settings
|
// UpdateUserSettings returns user settings
|
||||||
@@ -46,20 +51,26 @@ func UpdateUserSettings(ctx *context.APIContext) {
|
|||||||
form := web.GetForm(ctx).(*api.UserSettingsOptions)
|
form := web.GetForm(ctx).(*api.UserSettingsOptions)
|
||||||
|
|
||||||
opts := &user_service.UpdateOptions{
|
opts := &user_service.UpdateOptions{
|
||||||
FullName: optional.FromPtr(form.FullName),
|
FullName: optional.FromPtr(form.FullName),
|
||||||
Description: optional.FromPtr(form.Description),
|
Description: optional.FromPtr(form.Description),
|
||||||
Website: optional.FromPtr(form.Website),
|
Website: optional.FromPtr(form.Website),
|
||||||
Location: optional.FromPtr(form.Location),
|
Location: optional.FromPtr(form.Location),
|
||||||
Language: optional.FromPtr(form.Language),
|
Language: optional.FromPtr(form.Language),
|
||||||
Theme: optional.FromPtr(form.Theme),
|
Theme: optional.FromPtr(form.Theme),
|
||||||
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
|
DiffViewStyle: optional.FromPtr(form.DiffViewStyle),
|
||||||
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
|
KeepEmailPrivate: optional.FromPtr(form.HideEmail),
|
||||||
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
|
KeepActivityPrivate: optional.FromPtr(form.HideActivity),
|
||||||
|
IncludePrivateContributions: optional.FromPtr(form.IncludePrivateContributions),
|
||||||
}
|
}
|
||||||
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
|
if err := user_service.UpdateUser(ctx, ctx.Doer, opts); err != nil {
|
||||||
ctx.APIErrorInternal(err)
|
ctx.APIErrorInternal(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.JSON(http.StatusOK, convert.User2UserSettings(ctx.Doer))
|
settings, err := convert.User2UserSettings(ctx, ctx.Doer)
|
||||||
|
if err != nil {
|
||||||
|
ctx.APIErrorInternal(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.JSON(http.StatusOK, settings)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,16 +48,32 @@ func Profile(ctx *context.Context) {
|
|||||||
ctx.Data["PageIsSettingsProfile"] = true
|
ctx.Data["PageIsSettingsProfile"] = true
|
||||||
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
||||||
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
||||||
|
if !loadProfilePrivateContributionSetting(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadProfilePrivateContributionSetting(ctx *context.Context) bool {
|
||||||
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, ctx.Doer.ID)
|
||||||
|
if err != nil {
|
||||||
|
ctx.ServerError("GetIncludePrivateContributions", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ctx.Data["IncludePrivateContributions"] = includePrivateContributions
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// ProfilePost response for change user's profile
|
// ProfilePost response for change user's profile
|
||||||
func ProfilePost(ctx *context.Context) {
|
func ProfilePost(ctx *context.Context) {
|
||||||
ctx.Data["Title"] = ctx.Tr("settings_title")
|
ctx.Data["Title"] = ctx.Tr("settings_title")
|
||||||
ctx.Data["PageIsSettingsProfile"] = true
|
ctx.Data["PageIsSettingsProfile"] = true
|
||||||
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
|
||||||
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
ctx.Data["DisableGravatar"] = setting.Config().Picture.DisableGravatar.Value(ctx)
|
||||||
|
if !loadProfilePrivateContributionSetting(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if ctx.HasError() {
|
if ctx.HasError() {
|
||||||
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
ctx.HTML(http.StatusOK, tplSettingsProfile)
|
||||||
@@ -94,12 +110,13 @@ func ProfilePost(ctx *context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
opts := &user_service.UpdateOptions{
|
opts := &user_service.UpdateOptions{
|
||||||
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
|
KeepEmailPrivate: optional.Some(form.KeepEmailPrivate),
|
||||||
Description: optional.Some(form.Description),
|
Description: optional.Some(form.Description),
|
||||||
Website: optional.Some(form.Website),
|
Website: optional.Some(form.Website),
|
||||||
Location: optional.Some(form.Location),
|
Location: optional.Some(form.Location),
|
||||||
Visibility: optional.Some(form.Visibility),
|
Visibility: optional.Some(form.Visibility),
|
||||||
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
|
KeepActivityPrivate: optional.Some(form.KeepActivityPrivate),
|
||||||
|
IncludePrivateContributions: optional.Some(form.IncludePrivateContributions),
|
||||||
}
|
}
|
||||||
|
|
||||||
if form.FullName != "" {
|
if form.FullName != "" {
|
||||||
|
|||||||
@@ -86,18 +86,23 @@ func toUser(ctx context.Context, user *user_model.User, signed, authed bool) *ap
|
|||||||
}
|
}
|
||||||
|
|
||||||
// User2UserSettings return UserSettings based on a user
|
// User2UserSettings return UserSettings based on a user
|
||||||
func User2UserSettings(user *user_model.User) api.UserSettings {
|
func User2UserSettings(ctx context.Context, user *user_model.User) (api.UserSettings, error) {
|
||||||
return api.UserSettings{
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(ctx, user.ID)
|
||||||
FullName: user.FullName,
|
if err != nil {
|
||||||
Website: user.Website,
|
return api.UserSettings{}, err
|
||||||
Location: user.Location,
|
|
||||||
Language: user.Language,
|
|
||||||
Description: user.Description,
|
|
||||||
Theme: user.Theme,
|
|
||||||
HideEmail: user.KeepEmailPrivate,
|
|
||||||
HideActivity: user.KeepActivityPrivate,
|
|
||||||
DiffViewStyle: user.DiffViewStyle,
|
|
||||||
}
|
}
|
||||||
|
return api.UserSettings{
|
||||||
|
FullName: user.FullName,
|
||||||
|
Website: user.Website,
|
||||||
|
Location: user.Location,
|
||||||
|
Language: user.Language,
|
||||||
|
Description: user.Description,
|
||||||
|
Theme: user.Theme,
|
||||||
|
HideEmail: user.KeepEmailPrivate,
|
||||||
|
HideActivity: user.KeepActivityPrivate,
|
||||||
|
IncludePrivateContributions: includePrivateContributions,
|
||||||
|
DiffViewStyle: user.DiffViewStyle,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ToUserAndPermission return User and its collaboration permission for a repository
|
// ToUserAndPermission return User and its collaboration permission for a repository
|
||||||
|
|||||||
@@ -208,14 +208,15 @@ func (f *IntrospectTokenForm) Validate(req *http.Request, errs binding.Errors) b
|
|||||||
|
|
||||||
// UpdateProfileForm form for updating profile
|
// UpdateProfileForm form for updating profile
|
||||||
type UpdateProfileForm struct {
|
type UpdateProfileForm struct {
|
||||||
Name string `binding:"Username;MaxSize(40)"`
|
Name string `binding:"Username;MaxSize(40)"`
|
||||||
FullName string `binding:"MaxSize(100)"`
|
FullName string `binding:"MaxSize(100)"`
|
||||||
KeepEmailPrivate bool
|
KeepEmailPrivate bool
|
||||||
Website string `binding:"ValidSiteUrl;MaxSize(255)"`
|
Website string `binding:"ValidSiteUrl;MaxSize(255)"`
|
||||||
Location string `binding:"MaxSize(50)"`
|
Location string `binding:"MaxSize(50)"`
|
||||||
Description string `binding:"MaxSize(255)"`
|
Description string `binding:"MaxSize(255)"`
|
||||||
Visibility structs.VisibleType
|
Visibility structs.VisibleType
|
||||||
KeepActivityPrivate bool
|
KeepActivityPrivate bool
|
||||||
|
IncludePrivateContributions bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate validates the fields
|
// Validate validates the fields
|
||||||
|
|||||||
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
|||||||
defer gitRepo.Close()
|
defer gitRepo.Close()
|
||||||
|
|
||||||
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
|
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
|
||||||
|
indexHeatmap := false
|
||||||
if len(results) > 0 {
|
if len(results) > 0 {
|
||||||
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
|
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
|
||||||
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
|
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
|
||||||
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
|||||||
if result.RefName.IsPull() {
|
if result.RefName.IsPull() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
|
||||||
|
indexHeatmap = true
|
||||||
|
}
|
||||||
|
|
||||||
// Create reference
|
// Create reference
|
||||||
if result.OldCommitID == "" {
|
if result.OldCommitID == "" {
|
||||||
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
|
|||||||
NewCommitID: newCommitID,
|
NewCommitID: newCommitID,
|
||||||
}, theCommits)
|
}, theCommits)
|
||||||
}
|
}
|
||||||
|
if indexHeatmap {
|
||||||
|
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
|
||||||
|
log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
|
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
|
||||||
|
|
||||||
isEmpty, err := gitRepo.IsEmpty()
|
isEmpty, err := gitRepo.IsEmpty()
|
||||||
|
|||||||
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
|
|||||||
}
|
}
|
||||||
|
|
||||||
notify_service.ChangeDefaultBranch(ctx, repo)
|
notify_service.ChangeDefaultBranch(ctx, repo)
|
||||||
|
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return repo, nil
|
return repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/gitrepo"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
const heatmapIndexCommitsPageSize = 100
|
||||||
|
|
||||||
|
// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch.
|
||||||
|
func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error {
|
||||||
|
if repo == nil || repo.ID == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if repo.DefaultBranch == "" || repo.IsEmpty {
|
||||||
|
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer closer.Close()
|
||||||
|
|
||||||
|
head, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
|
||||||
|
if git.IsErrNotExist(err) {
|
||||||
|
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0)
|
||||||
|
now := timeutil.TimeStampNow()
|
||||||
|
for page := 1; ; page++ {
|
||||||
|
commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(commits) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, commit := range commits {
|
||||||
|
commitSHA := commit.ID.String()
|
||||||
|
if commit.Author == nil || commit.Author.Email == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
authorUnix := timeutil.TimeStamp(commit.Author.When.Unix())
|
||||||
|
if authorUnix > now {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repo.ID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: strings.ToLower(commit.Author.Email),
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{
|
||||||
|
RepoID: repo.ID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
UserID: userID,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(commits) < heatmapIndexCommitsPageSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) {
|
||||||
|
address, err := user_model.GetEmailAddressByEmail(ctx, email)
|
||||||
|
if user_model.IsErrEmailAddressNotExist(err) {
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, err
|
||||||
|
}
|
||||||
|
if !address.IsActivated {
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
return address.UID, true, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||||
|
repo_module "code.gitea.io/gitea/modules/repository"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
{Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
{Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
|
||||||
|
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, int64(2), contributions[0].UserID)
|
||||||
|
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
|
||||||
|
assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail)
|
||||||
|
assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix)
|
||||||
|
|
||||||
|
emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com")
|
||||||
|
require.NoError(t, err)
|
||||||
|
emailAddress.UID = 1
|
||||||
|
require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress))
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
|
||||||
|
contributions = loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 1)
|
||||||
|
assert.Equal(t, int64(1), contributions[0].UserID)
|
||||||
|
assert.Equal(t, commits["old"], contributions[0].CommitSHA)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
{Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
{Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
|
||||||
|
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 2)
|
||||||
|
assert.Equal(t, int64(2), contributions[0].UserID)
|
||||||
|
assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA)
|
||||||
|
assert.Equal(t, int64(1), contributions[1].UserID)
|
||||||
|
assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA)
|
||||||
|
for _, contribution := range contributions {
|
||||||
|
assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA)
|
||||||
|
}
|
||||||
|
|
||||||
|
deleteMainRef(t, repo)
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
|
||||||
|
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
|
||||||
|
})
|
||||||
|
|
||||||
|
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
|
||||||
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
|
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
|
||||||
|
assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
|
||||||
|
|
||||||
|
newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
|
||||||
|
{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
|
||||||
|
})
|
||||||
|
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
|
||||||
|
{
|
||||||
|
RefFullName: git.RefNameFromBranch("main"),
|
||||||
|
OldCommitID: commits["initial"],
|
||||||
|
NewCommitID: newCommits["pushed"],
|
||||||
|
PusherID: 1,
|
||||||
|
RepoUserName: repo.OwnerName,
|
||||||
|
RepoName: repo.Name,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
|
||||||
|
require.Len(t, contributions, 2)
|
||||||
|
assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
|
||||||
|
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
|
||||||
|
{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
|
||||||
|
})
|
||||||
|
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
|
||||||
|
{
|
||||||
|
RefFullName: git.RefNameFromBranch("feature"),
|
||||||
|
OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(),
|
||||||
|
NewCommitID: featureCommits["feature-pushed"],
|
||||||
|
PusherID: 1,
|
||||||
|
RepoUserName: repo.OwnerName,
|
||||||
|
RepoName: repo.Name,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
type heatmapIndexTestCommit struct {
|
||||||
|
Branch string
|
||||||
|
Mark string
|
||||||
|
Parent string
|
||||||
|
AuthorName string
|
||||||
|
AuthorEmail string
|
||||||
|
CommitterName string
|
||||||
|
CommitterEmail string
|
||||||
|
AuthorDate string
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
|
||||||
|
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
|
repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
marks := runFastImport(t, repo, commits)
|
||||||
|
repo.IsEmpty = false
|
||||||
|
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
|
||||||
|
return repo, marks
|
||||||
|
}
|
||||||
|
|
||||||
|
func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var stream strings.Builder
|
||||||
|
branchTips := make(map[string]string)
|
||||||
|
for i, commit := range commits {
|
||||||
|
mark := fmt.Sprintf(":%d", i+1)
|
||||||
|
branchRef := "refs/heads/" + commit.Branch
|
||||||
|
stream.WriteString("commit " + branchRef + "\n")
|
||||||
|
stream.WriteString("mark " + mark + "\n")
|
||||||
|
stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
|
||||||
|
stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
|
||||||
|
message := "heatmap " + commit.Mark
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
|
||||||
|
if parentMark := branchTips[commit.Branch]; parentMark != "" {
|
||||||
|
stream.WriteString("from " + parentMark + "\n")
|
||||||
|
} else if commit.Parent != "" {
|
||||||
|
stream.WriteString("from " + commit.Parent + "\n")
|
||||||
|
}
|
||||||
|
content := commit.Mark + "\n"
|
||||||
|
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
|
||||||
|
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
|
||||||
|
branchTips[commit.Branch] = mark
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
WithStdinCopy(strings.NewReader(stream.String())).
|
||||||
|
Run(t.Context()))
|
||||||
|
|
||||||
|
commitSHAs := make(map[string]string, len(commits))
|
||||||
|
for _, commit := range commits {
|
||||||
|
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
|
||||||
|
AddOptionFormat("--grep=%s", "heatmap "+commit.Mark).
|
||||||
|
AddDynamicArguments("refs/heads/" + commit.Branch).
|
||||||
|
WithDir(repo.RepoPath()).
|
||||||
|
RunStdString(t.Context())
|
||||||
|
require.NoError(t, err)
|
||||||
|
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
|
||||||
|
}
|
||||||
|
return commitSHAs
|
||||||
|
}
|
||||||
|
|
||||||
|
func signatureLine(kind, name, email, date string) string {
|
||||||
|
parsed, err := time.Parse(time.RFC3339, date)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
|
||||||
|
t.Helper()
|
||||||
|
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return contributions
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteMainRef(t *testing.T, repo *repo_model.Repository) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
|
||||||
|
}
|
||||||
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
|
repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
|
||||||
if opts.Mirror {
|
if opts.Mirror {
|
||||||
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
|
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
|||||||
}
|
}
|
||||||
return repo, nil
|
return repo, nil
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
|
||||||
|
}
|
||||||
|
return repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
|
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
indexHeatmap := false
|
||||||
if !opts.IsDelRef() {
|
if !opts.IsDelRef() {
|
||||||
branch := opts.RefFullName.BranchName()
|
branch := opts.RefFullName.BranchName()
|
||||||
|
|
||||||
@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
|||||||
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
|
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
|
||||||
log.Error("DelRepoDivergenceFromCache: %v", err)
|
log.Error("DelRepoDivergenceFromCache: %v", err)
|
||||||
}
|
}
|
||||||
|
indexHeatmap = true
|
||||||
} else {
|
} else {
|
||||||
if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
|
if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
|
||||||
log.Error("DelDivergenceFromCache: %v", err)
|
log.Error("DelDivergenceFromCache: %v", err)
|
||||||
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
|||||||
pushDeleteBranch(ctx, repo, pusher, opts)
|
pushDeleteBranch(ctx, repo, pusher, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if indexHeatmap {
|
||||||
|
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
|
||||||
|
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
||||||
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
|
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
|
||||||
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
|
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
auth_model "code.gitea.io/gitea/models/auth"
|
auth_model "code.gitea.io/gitea/models/auth"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
password_module "code.gitea.io/gitea/modules/auth/password"
|
password_module "code.gitea.io/gitea/modules/auth/password"
|
||||||
"code.gitea.io/gitea/modules/optional"
|
"code.gitea.io/gitea/modules/optional"
|
||||||
@@ -47,6 +48,7 @@ type UpdateOptions struct {
|
|||||||
IsRestricted optional.Option[bool]
|
IsRestricted optional.Option[bool]
|
||||||
Visibility optional.Option[structs.VisibleType]
|
Visibility optional.Option[structs.VisibleType]
|
||||||
KeepActivityPrivate optional.Option[bool]
|
KeepActivityPrivate optional.Option[bool]
|
||||||
|
IncludePrivateContributions optional.Option[bool]
|
||||||
Language optional.Option[string]
|
Language optional.Option[string]
|
||||||
Theme optional.Option[string]
|
Theme optional.Option[string]
|
||||||
DiffViewStyle optional.Option[string]
|
DiffViewStyle optional.Option[string]
|
||||||
@@ -182,7 +184,23 @@ func UpdateUser(ctx context.Context, u *user_model.User, opts *UpdateOptions) er
|
|||||||
cols = append(cols, "last_login_unix")
|
cols = append(cols, "last_login_unix")
|
||||||
}
|
}
|
||||||
|
|
||||||
return user_model.UpdateUserCols(ctx, u, cols...)
|
if len(cols) > 0 || opts.IncludePrivateContributions.Has() {
|
||||||
|
return db.WithTx(ctx, func(ctx context.Context) error {
|
||||||
|
if len(cols) > 0 {
|
||||||
|
if err := user_model.UpdateUserCols(ctx, u, cols...); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.IncludePrivateContributions.Has() {
|
||||||
|
return user_model.SetIncludePrivateContributions(ctx, u.ID, opts.IncludePrivateContributions.Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type UpdateAuthOptions struct {
|
type UpdateAuthOptions struct {
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ func TestUpdateUser(t *testing.T) {
|
|||||||
IsAdmin: UpdateOptionFieldFromValue(true),
|
IsAdmin: UpdateOptionFieldFromValue(true),
|
||||||
Visibility: optional.Some(structs.VisibleTypePrivate),
|
Visibility: optional.Some(structs.VisibleTypePrivate),
|
||||||
KeepActivityPrivate: optional.Some(true),
|
KeepActivityPrivate: optional.Some(true),
|
||||||
|
IncludePrivateContributions: optional.Some(true),
|
||||||
Language: optional.Some("lang"),
|
Language: optional.Some("lang"),
|
||||||
Theme: optional.Some("theme"),
|
Theme: optional.Some("theme"),
|
||||||
DiffViewStyle: optional.Some("split"),
|
DiffViewStyle: optional.Some("split"),
|
||||||
@@ -66,6 +67,9 @@ func TestUpdateUser(t *testing.T) {
|
|||||||
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
||||||
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
||||||
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
||||||
|
includePrivateContributions, err := user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
|
||||||
assert.Equal(t, opts.Language.Value(), user.Language)
|
assert.Equal(t, opts.Language.Value(), user.Language)
|
||||||
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
||||||
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
||||||
@@ -86,6 +90,9 @@ func TestUpdateUser(t *testing.T) {
|
|||||||
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin)
|
||||||
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
assert.Equal(t, opts.Visibility.Value(), user.Visibility)
|
||||||
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate)
|
||||||
|
includePrivateContributions, err = user_model.GetIncludePrivateContributions(t.Context(), user.ID)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, opts.IncludePrivateContributions.Value(), includePrivateContributions)
|
||||||
assert.Equal(t, opts.Language.Value(), user.Language)
|
assert.Equal(t, opts.Language.Value(), user.Language)
|
||||||
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
assert.Equal(t, opts.Theme.Value(), user.Theme)
|
||||||
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
assert.Equal(t, opts.DiffViewStyle.Value(), user.DiffViewStyle)
|
||||||
|
|||||||
+9
-1
@@ -29655,6 +29655,10 @@
|
|||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
"x-go-name": "HideEmail"
|
"x-go-name": "HideEmail"
|
||||||
},
|
},
|
||||||
|
"include_private_contributions": {
|
||||||
|
"type": "boolean",
|
||||||
|
"x-go-name": "IncludePrivateContributions"
|
||||||
|
},
|
||||||
"language": {
|
"language": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"x-go-name": "Language"
|
"x-go-name": "Language"
|
||||||
@@ -29699,6 +29703,10 @@
|
|||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
"x-go-name": "HideEmail"
|
"x-go-name": "HideEmail"
|
||||||
},
|
},
|
||||||
|
"include_private_contributions": {
|
||||||
|
"type": "boolean",
|
||||||
|
"x-go-name": "IncludePrivateContributions"
|
||||||
|
},
|
||||||
"language": {
|
"language": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"x-go-name": "Language"
|
"x-go-name": "Language"
|
||||||
@@ -31073,4 +31081,4 @@
|
|||||||
"TOTPHeader": []
|
"TOTPHeader": []
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,6 +88,13 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="field">
|
||||||
|
<div class="ui checkbox" id="include-private-contributions">
|
||||||
|
<label data-tooltip-content="{{ctx.Locale.Tr "settings.include_private_contributions_popup"}}"><strong>{{ctx.Locale.Tr "settings.include_private_contributions"}}</strong></label>
|
||||||
|
<input name="include_private_contributions" type="checkbox" {{if .IncludePrivateContributions}}checked{{end}}>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="divider"></div>
|
<div class="divider"></div>
|
||||||
|
|
||||||
<div class="field">
|
<div class="field">
|
||||||
|
|||||||
@@ -4,17 +4,19 @@
|
|||||||
package integration
|
package integration
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
activities_model "code.gitea.io/gitea/models/activities"
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
auth_model "code.gitea.io/gitea/models/auth"
|
auth_model "code.gitea.io/gitea/models/auth"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
"code.gitea.io/gitea/tests"
|
"code.gitea.io/gitea/tests"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestUserHeatmap(t *testing.T) {
|
func TestUserHeatmap(t *testing.T) {
|
||||||
@@ -23,17 +25,23 @@ func TestUserHeatmap(t *testing.T) {
|
|||||||
normalUsername := "user2"
|
normalUsername := "user2"
|
||||||
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
|
token := getUserToken(t, adminUsername, auth_model.AccessTokenScopeReadUser)
|
||||||
|
|
||||||
fakeNow := time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local)
|
fakeNow := time.Date(2011, 10, 21, 0, 0, 0, 0, time.Local)
|
||||||
timeutil.MockSet(fakeNow)
|
timeutil.MockSet(fakeNow)
|
||||||
defer timeutil.MockUnset()
|
defer timeutil.MockUnset()
|
||||||
|
|
||||||
req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)).
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-author-date", 1319068800)
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "api-user2-public-same-bucket", 1319068850)
|
||||||
|
testHeatmapSeedContribution(t, 2, 2, "api-user2-private-hidden", 1319070600)
|
||||||
|
|
||||||
|
req := NewRequestf(t, "GET", "/api/v1/users/%s/heatmap", normalUsername).
|
||||||
AddTokenAuth(token)
|
AddTokenAuth(token)
|
||||||
resp := MakeRequest(t, req, http.StatusOK)
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
var heatmap []*activities_model.UserHeatmapData
|
var heatmap []*activities_model.UserHeatmapData
|
||||||
DecodeJSON(t, resp, &heatmap)
|
DecodeJSON(t, resp, &heatmap)
|
||||||
var dummyheatmap []*activities_model.UserHeatmapData
|
|
||||||
dummyheatmap = append(dummyheatmap, &activities_model.UserHeatmapData{Timestamp: 1603227600, Contributions: 1})
|
|
||||||
|
|
||||||
assert.Equal(t, dummyheatmap, heatmap)
|
assert.Equal(t, []*activities_model.UserHeatmapData{
|
||||||
|
{Timestamp: 1319068800, Contributions: 2},
|
||||||
|
}, heatmap)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,22 +4,40 @@
|
|||||||
package integration
|
package integration
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha1"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
activities_model "code.gitea.io/gitea/models/activities"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
"code.gitea.io/gitea/modules/json"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
"code.gitea.io/gitea/tests"
|
"code.gitea.io/gitea/tests"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type testHeatmapWebResponse struct {
|
||||||
|
HeatmapData [][2]int64 `json:"heatmapData"`
|
||||||
|
TotalContributions int64 `json:"totalContributions"`
|
||||||
|
}
|
||||||
|
|
||||||
func TestHeatmapEndpoints(t *testing.T) {
|
func TestHeatmapEndpoints(t *testing.T) {
|
||||||
defer tests.PrepareTestEnv(t)()
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
// Mock time so fixture actions fall within the heatmap's time window
|
// Mock time so fixture actions fall within the heatmap's time window
|
||||||
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
timeutil.MockSet(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||||
defer timeutil.MockUnset()
|
defer timeutil.MockUnset()
|
||||||
|
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
|
||||||
|
require.NoError(t, user_model.SetIncludePrivateContributions(t.Context(), 2, false))
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-one", 1603009800)
|
||||||
|
testHeatmapSeedContribution(t, 2, 1, "web-user2-public-two", 1603009850)
|
||||||
|
testHeatmapSeedContribution(t, 2, 2, "web-user2-private-hidden", 1603010700)
|
||||||
|
|
||||||
session := loginUser(t, "user2")
|
session := loginUser(t, "user2")
|
||||||
|
|
||||||
@@ -28,11 +46,15 @@ func TestHeatmapEndpoints(t *testing.T) {
|
|||||||
req := NewRequest(t, "GET", "/user2/-/heatmap")
|
req := NewRequest(t, "GET", "/user2/-/heatmap")
|
||||||
resp := session.MakeRequest(t, req, http.StatusOK)
|
resp := session.MakeRequest(t, req, http.StatusOK)
|
||||||
|
|
||||||
var result map[string]any
|
webHeatmap := testHeatmapDecodeWebResponse(t, resp)
|
||||||
DecodeJSON(t, resp, &result)
|
|
||||||
assert.Contains(t, result, "heatmapData")
|
req = NewRequest(t, "GET", "/api/v1/users/user2/heatmap")
|
||||||
assert.Contains(t, result, "totalContributions")
|
resp = session.MakeRequest(t, req, http.StatusOK)
|
||||||
assert.Positive(t, result["totalContributions"])
|
var apiHeatmap []*activities_model.UserHeatmapData
|
||||||
|
DecodeJSON(t, resp, &apiHeatmap)
|
||||||
|
|
||||||
|
assert.Equal(t, testHeatmapSumAPIContributions(apiHeatmap), webHeatmap.TotalContributions)
|
||||||
|
assert.Equal(t, int64(2), webHeatmap.TotalContributions)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("OrgDashboard", func(t *testing.T) {
|
t.Run("OrgDashboard", func(t *testing.T) {
|
||||||
@@ -57,3 +79,71 @@ func TestHeatmapEndpoints(t *testing.T) {
|
|||||||
assert.Contains(t, result, "totalContributions")
|
assert.Contains(t, result, "totalContributions")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testHeatmapSeedContribution(t *testing.T, userID, repoID int64, label string, authorUnix timeutil.TimeStamp) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
commitSHA := fmt.Sprintf("%040x", sha1.Sum([]byte(label)))
|
||||||
|
require.NoError(t, activities_model.UpsertHeatmapContribution(t.Context(), &activities_model.HeatmapContribution{
|
||||||
|
UserID: userID,
|
||||||
|
RepoID: repoID,
|
||||||
|
CommitSHA: commitSHA,
|
||||||
|
AuthorEmail: fmt.Sprintf("user%d@example.com", userID),
|
||||||
|
AuthorUnix: authorUnix,
|
||||||
|
}))
|
||||||
|
return commitSHA
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapDecodeWebResponse(t *testing.T, resp *httptest.ResponseRecorder) testHeatmapWebResponse {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var result testHeatmapWebResponse
|
||||||
|
DecodeJSON(t, resp, &result)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapSumAPIContributions(heatmap []*activities_model.UserHeatmapData) int64 {
|
||||||
|
var total int64
|
||||||
|
for _, item := range heatmap {
|
||||||
|
total += item.Contributions
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapAssertAggregateOnlyAPIResponse(t *testing.T, body []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var decoded []map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(body, &decoded))
|
||||||
|
for _, entry := range decoded {
|
||||||
|
assert.ElementsMatch(t, []string{"timestamp", "contributions"}, testHeatmapMapKeys(entry))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapAssertAggregateOnlyWebResponse(t *testing.T, body []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(body, &decoded))
|
||||||
|
assert.ElementsMatch(t, []string{"heatmapData", "totalContributions"}, testHeatmapMapKeys(decoded))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapAssertNoPrivateMetadata(t *testing.T, body []byte, forbidden ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
response := string(body)
|
||||||
|
for _, value := range forbidden {
|
||||||
|
assert.NotContains(t, response, value)
|
||||||
|
}
|
||||||
|
for _, value := range []string{"repo_id", "repository_id", "repoID", "commit_sha", "commit_message", "branch", "url", "action_id"} {
|
||||||
|
assert.NotContains(t, response, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testHeatmapMapKeys[K comparable, V any](m map[K]V) []K {
|
||||||
|
keys := make([]K, 0, len(m))
|
||||||
|
for key := range m {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user