feat: neuro: stable video diffusion

lib/default.nix

@@ -26,12 +26,39 @@
     "aarch64-darwin"
   ];
 
+  cudaUnfreeNames = [
+    "cuda_nvcc"
+    "cuda_cudart"
+    "cuda_cuobjdump"
+    "cuda_cupti"
+    "cuda_nvdisasm"
+    "cuda_cccl"
+    "cuda_nvml_dev"
+    "cuda_nvrtc"
+    "cuda_nvtx"
+    "cuda_profiler_api"
+
+    "libcusparse_lt"
+    "libcublas"
+    "libcufft"
+    "libcufile"
+    "libcurand"
+    "libcusolver"
+    "libnvjitlink"
+    "libcusparse"
+    "cudnn"
+  ];
+
+  cudaUnfreePredicate = pkg:
+    builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
+
   forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
     builtins.foldl' (
       acc: system: let
         pkgs = import nixpkgs {
           inherit system;
           overlays = pkgOverlays;
+          config.allowUnfreePredicate = cudaUnfreePredicate;
         };
         systemOutputs = f {
           system = system;
@@ -58,7 +85,7 @@
     else {};
 in {
   # -- For all systems --
-  inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems;
+  inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
 
   forSystems = systems: nixpkgs.lib.genAttrs systems;
   forAllSystems = nixpkgs.lib.genAttrs AllSystems;

nixos/module/hectic/service/stable-video-diffusion.nix

@@ -0,0 +1,153 @@
+{ ... }: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.stable-video-diffusion;
+in {
+  options.hectic.services.stable-video-diffusion = {
+    enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
+
+    host = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1";
+      description = "Address the Stable Video Diffusion API binds to.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 7861;
+      description = "Port the Stable Video Diffusion API binds to.";
+    };
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.hectic.stable-video-diffusion-api;
+      defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
+      description = "Package providing the Stable Video Diffusion API executable.";
+    };
+
+    modelId = lib.mkOption {
+      type = lib.types.str;
+      default = "stabilityai/stable-video-diffusion-img2vid-xt";
+      description = "Model identifier loaded by the Stable Video Diffusion API.";
+    };
+
+    device = lib.mkOption {
+      type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
+      default = null;
+      description = ''
+        Torch device requested from the Stable Video Diffusion API. When null,
+        the package keeps its own auto-detection behavior.
+      '';
+    };
+
+    libraryPath = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [];
+      description = ''
+        Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
+        should include /run/opengl-driver/lib so libcuda.so is visible to torch.
+      '';
+    };
+
+    stateDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/stable-video-diffusion-api";
+      description = "Persistent state directory for the Stable Video Diffusion API.";
+    };
+
+    cacheDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/cache/stable-video-diffusion-api";
+      description = "Cache directory for downloaded model and runtime artifacts.";
+    };
+
+    outputDir = lib.mkOption {
+      type = lib.types.str;
+      default = "${cfg.stateDir}/outputs";
+      defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
+      description = "Directory where generated video outputs are written.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = with lib.types; nullOr path;
+      default = null;
+      description = ''
+        Optional environment file for secrets or runtime overrides. Values from
+        the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
+        SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
+        and LD_LIBRARY_PATH by default.
+      '';
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Whether to open the API port in the firewall.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.device != "cuda" || cfg.libraryPath != [];
+        message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
+      }
+    ];
+
+    users.users.stable-video-diffusion = {
+      isSystemUser = true;
+      group = "stable-video-diffusion";
+    };
+    users.groups.stable-video-diffusion = {};
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
+      "d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
+      "d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
+    ];
+
+    systemd.services.stable-video-diffusion-api = {
+      description = "Stable Video Diffusion HTTP API";
+      after       = [ "network.target" ];
+      wantedBy    = [ "multi-user.target" ];
+      environment = {
+        SVD_API_HOST   = cfg.host;
+        SVD_API_PORT   = toString cfg.port;
+        SVD_MODEL_ID   = cfg.modelId;
+        SVD_STATE_DIR  = cfg.stateDir;
+        SVD_CACHE_DIR  = cfg.cacheDir;
+        SVD_OUTPUT_DIR = cfg.outputDir;
+      }
+      // lib.optionalAttrs (cfg.device != null) {
+        SVD_DEVICE = cfg.device;
+      }
+      // lib.optionalAttrs (cfg.libraryPath != []) {
+        LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
+      };
+      serviceConfig = lib.mkMerge [
+        {
+          Type            = "simple";
+          User            = "stable-video-diffusion";
+          Group           = "stable-video-diffusion";
+          WorkingDirectory = cfg.stateDir;
+          ExecStart       = lib.getExe' cfg.package "stable-video-diffusion-api";
+          Restart         = "on-failure";
+          RestartSec      = "5s";
+          TimeoutStopSec  = "30s";
+          KillSignal      = "SIGTERM";
+          KillMode        = "mixed";
+          StandardOutput  = "journal";
+          StandardError   = "journal";
+        }
+        (lib.mkIf (cfg.environmentFile != null) {
+          EnvironmentFile = cfg.environmentFile;
+        })
+      ];
+    };
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
+  };
+}

nixos/system/neuro/default.nix

@@ -15,32 +15,12 @@ in self.lib.nixpkgs-lib.nixosSystem {
       self.overlays.default
       inputs.nix-minecraft.overlay
     ];
-    config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
+    config.allowUnfreePredicate = pkg:
+      self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
       "minecraft-server"
       "neoforge"
 
       "nvidia-x11"
-
-      "cuda_nvcc"
-      "cuda_cudart"
-      "cuda_cuobjdump"
-      "cuda_cupti"
-      "cuda_nvdisasm"
-      "cuda_cccl"
-      "cuda_nvml_dev"
-      "cuda_nvrtc"
-      "cuda_nvtx"
-      "cuda_profiler_api"
-
-      "libcusparse_lt"
-      "libcublas"
-      "libcufft"
-      "libcufile"
-      "libcurand"
-      "libcusolver"
-      "libnvjitlink"
-      "libcusparse"
-      "cudnn"
     ];
     # jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
     config.permittedInsecurePackages = [

nixos/system/neuro/neuro.nix

@@ -17,6 +17,11 @@
 
   ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";
 
+  stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
+    pkgs.stdenv.cc.cc.lib
+    pkgs.zlib
+  ];
+
   ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
     pname = "ollama";
     version = "0.22.1";
@@ -176,6 +181,19 @@ in {
     openFirewall = false;
   };
 
+  hectic.services.stable-video-diffusion = {
+    enable       = true;
+    host         = "127.0.0.1";
+    port         = 7861;
+    package      = pkgs.hectic.stable-video-diffusion-api;
+    device       = "cuda";
+    libraryPath  = [
+      stableVideoDiffusionLibraryPath
+      "/run/opengl-driver/lib"
+    ];
+    openFirewall = false;
+  };
+
   networking = {
     networkmanager.enable = true;
     useDHCP = lib.mkDefault true;

package/default.nix

@@ -174,5 +174,6 @@ in {
   pg-15-ext-smtp-client        = buildSmtpExt      pkgs "15";
   pg-15-ext-plhaskell          = buildPlHaskellExt pkgs "15";
   pg-15-ext-plsh               = buildPlShExt      pkgs "15";
+  stable-video-diffusion-api   = pkgs.callPackage ./stable-video-diffusion-api {};
   media-browser                = pkgs.callPackage ./media-browser {};
 }

package/stable-video-diffusion-api/app.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Local HTTP API for Diffusers Stable Video Diffusion image-to-video."""
+
+import os
+import threading
+import uuid
+from importlib import import_module
+from pathlib import Path
+from typing import Any
+
+torch = import_module("torch")
+uvicorn = import_module("uvicorn")
+diffusers = import_module("diffusers")
+diffusers_utils = import_module("diffusers.utils")
+fastapi = import_module("fastapi")
+pil_image = import_module("PIL.Image")
+pil_unidentified_image_error = import_module("PIL").UnidentifiedImageError
+
+FastAPI = fastapi.FastAPI
+File = fastapi.File
+Form = fastapi.Form
+HTTPException = fastapi.HTTPException
+Request = fastapi.Request
+UploadFile = fastapi.UploadFile
+StableVideoDiffusionPipeline = diffusers.StableVideoDiffusionPipeline
+export_to_video = diffusers_utils.export_to_video
+
+
+HOST = os.environ.get("SVD_API_HOST", "127.0.0.1")
+PORT = int(os.environ.get("SVD_API_PORT", "8000"))
+MODEL_ID = os.environ.get("SVD_MODEL_ID", "stabilityai/stable-video-diffusion-img2vid-xt")
+CACHE_DIR = os.environ.get("SVD_CACHE_DIR") or os.environ.get("HF_HOME")
+OUTPUT_DIR = Path(os.environ.get("SVD_OUTPUT_DIR", "/var/lib/stable-video-diffusion-api/outputs"))
+DEVICE = os.environ.get("SVD_DEVICE", "cuda" if torch.cuda.is_available() else "cpu")
+DTYPE = os.environ.get("SVD_DTYPE", "float16")
+ENABLE_CPU_OFFLOAD = os.environ.get("SVD_ENABLE_CPU_OFFLOAD", "0").lower() in ("1", "true", "yes", "on")
+DEFAULT_DECODE_CHUNK_SIZE = int(os.environ.get("SVD_DECODE_CHUNK_SIZE", "8"))
+DEFAULT_FPS = int(os.environ.get("SVD_FPS", "7"))
+
+
+app = FastAPI(title="Stable Video Diffusion API")
+pipeline_lock = threading.Lock()
+pipeline: Any | None = None
+
+
+def torch_dtype() -> Any:
+  dtypes = {
+    "float16": torch.float16,
+    "fp16": torch.float16,
+    "float32": torch.float32,
+    "fp32": torch.float32,
+    "bfloat16": torch.bfloat16,
+    "bf16": torch.bfloat16,
+  }
+  try:
+    return dtypes[DTYPE.lower()]
+  except KeyError as exc:
+    raise RuntimeError(f"Unsupported SVD_DTYPE: {DTYPE}") from exc
+
+
+def get_pipeline() -> Any:
+  global pipeline
+
+  if pipeline is not None:
+    return pipeline
+
+  with pipeline_lock:
+    if pipeline is not None:
+      return pipeline
+
+    kwargs: dict[str, Any] = {"torch_dtype": torch_dtype()}
+    if CACHE_DIR:
+      kwargs["cache_dir"] = CACHE_DIR
+
+    loaded = StableVideoDiffusionPipeline.from_pretrained(MODEL_ID, **kwargs)
+    if ENABLE_CPU_OFFLOAD:
+      loaded.enable_model_cpu_offload()
+    else:
+      loaded.to(DEVICE)
+
+    pipeline = loaded
+    return loaded
+
+
+def positive_int(name: str, value: Any, default: int) -> int:
+  if value in (None, ""):
+    return default
+  try:
+    parsed = int(value)
+  except (TypeError, ValueError) as exc:
+    raise HTTPException(status_code=400, detail=f"{name} must be an integer") from exc
+  if parsed <= 0:
+    raise HTTPException(status_code=400, detail=f"{name} must be greater than zero")
+  return parsed
+
+
+def optional_int(name: str, value: Any) -> int | None:
+  if value in (None, ""):
+    return None
+  try:
+    return int(value)
+  except (TypeError, ValueError) as exc:
+    raise HTTPException(status_code=400, detail=f"{name} must be an integer") from exc
+
+
+def optional_float(name: str, value: Any) -> float | None:
+  if value in (None, ""):
+    return None
+  try:
+    return float(value)
+  except (TypeError, ValueError) as exc:
+    raise HTTPException(status_code=400, detail=f"{name} must be a number") from exc
+
+
+def load_image_from_path(input_image_path: str) -> Any:
+  path = Path(input_image_path).expanduser()
+  if not path.exists() or not path.is_file():
+    raise HTTPException(status_code=400, detail="input_image_path does not exist or is not a file")
+  try:
+    return pil_image.open(path).convert("RGB")
+  except (OSError, pil_unidentified_image_error) as exc:
+    raise HTTPException(status_code=400, detail="input_image_path could not be decoded as an image") from exc
+
+
+async def load_uploaded_image(image: UploadFile) -> Any:
+  try:
+    return pil_image.open(image.file).convert("RGB")
+  except (OSError, pil_unidentified_image_error) as exc:
+    raise HTTPException(status_code=400, detail="uploaded image could not be decoded") from exc
+
+
+async def json_payload(request: Request) -> dict[str, Any]:
+  content_type = request.headers.get("content-type", "")
+  if not content_type.startswith("application/json"):
+    return {}
+  try:
+    payload = await request.json()
+  except ValueError as exc:
+    raise HTTPException(status_code=400, detail="request body must be valid JSON") from exc
+  if not isinstance(payload, dict):
+    raise HTTPException(status_code=400, detail="JSON request body must be an object")
+  return payload
+
+
+@app.get("/health")
+def health() -> dict[str, Any]:
+  return {
+    "status": "ok",
+    "model_loaded": pipeline is not None,
+    "model_id": MODEL_ID,
+    "device": DEVICE,
+    "dtype": DTYPE,
+    "cpu_offload": ENABLE_CPU_OFFLOAD,
+    "cache_dir": CACHE_DIR,
+    "output_dir": str(OUTPUT_DIR),
+  }
+
+
+@app.post("/generate")
+async def generate(
+  request: Request,
+  image: UploadFile | None = File(default=None),
+  input_image_path: str | None = Form(default=None),
+  num_frames: int | None = Form(default=None),
+  num_inference_steps: int | None = Form(default=None),
+  fps: int | None = Form(default=None),
+  decode_chunk_size: int | None = Form(default=None),
+  seed: int | None = Form(default=None),
+  motion_bucket_id: int | None = Form(default=None),
+  noise_aug_strength: float | None = Form(default=None),
+) -> dict[str, Any]:
+  payload = await json_payload(request)
+  path = input_image_path or payload.get("input_image_path")
+  upload = image
+
+  if upload is None and not path:
+    raise HTTPException(status_code=400, detail="provide input_image_path or upload image")
+  if upload is not None and path:
+    raise HTTPException(status_code=400, detail="provide only one image source")
+
+  source_image = await load_uploaded_image(upload) if upload is not None else load_image_from_path(str(path))
+  chunk_size = positive_int("decode_chunk_size", decode_chunk_size if decode_chunk_size is not None else payload.get("decode_chunk_size"), DEFAULT_DECODE_CHUNK_SIZE)
+  output_fps = positive_int("fps", fps if fps is not None else payload.get("fps"), DEFAULT_FPS)
+  request_seed = optional_int("seed", seed if seed is not None else payload.get("seed"))
+  generator = None
+  if request_seed is not None:
+    generator = torch.Generator(device="cpu").manual_seed(request_seed)
+
+  call_args: dict[str, Any] = {
+    "image": source_image,
+    "decode_chunk_size": chunk_size,
+  }
+  for key, value in {
+    "num_frames": num_frames if num_frames is not None else payload.get("num_frames"),
+    "num_inference_steps": num_inference_steps if num_inference_steps is not None else payload.get("num_inference_steps"),
+    "motion_bucket_id": motion_bucket_id if motion_bucket_id is not None else payload.get("motion_bucket_id"),
+  }.items():
+    parsed = optional_int(key, value)
+    if parsed is not None:
+      call_args[key] = parsed
+
+  parsed_noise = optional_float("noise_aug_strength", noise_aug_strength if noise_aug_strength is not None else payload.get("noise_aug_strength"))
+  if parsed_noise is not None:
+    call_args["noise_aug_strength"] = parsed_noise
+  if generator is not None:
+    call_args["generator"] = generator
+
+  OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+  output_path = OUTPUT_DIR / f"svd-{uuid.uuid4().hex}.mp4"
+
+  try:
+    frames = get_pipeline()(**call_args).frames[0]
+    export_to_video(frames, str(output_path), fps=output_fps)
+  except Exception as exc:
+    raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+  return {
+    "output_path": str(output_path),
+    "model_id": MODEL_ID,
+    "seed": request_seed,
+    "fps": output_fps,
+  }
+
+
+def main() -> None:
+  uvicorn.run("app:app", host=HOST, port=PORT)
+
+
+if __name__ == "__main__":
+  main()

package/stable-video-diffusion-api/default.nix

@@ -0,0 +1,41 @@
+{ pkgs }:
+
+let
+  pythonEnv = pkgs.python3.withPackages (ps: let
+    torch = ps.torchWithCuda;
+    accelerate = ps.accelerate.override { inherit torch; };
+  in [
+    accelerate
+    ps.diffusers
+    ps.fastapi
+    ps.imageio
+    ps.imageio-ffmpeg
+    ps.pillow
+    ps.python-multipart
+    ps.safetensors
+    torch
+    ps.transformers
+    ps.uvicorn
+  ]);
+in
+
+pkgs.stdenv.mkDerivation {
+  pname = "stable-video-diffusion-api";
+  version = "0.1.0";
+  src = ./.;
+
+  nativeBuildInputs = [ pkgs.makeWrapper ];
+
+  installPhase = ''
+    mkdir -p $out/bin $out/libexec/stable-video-diffusion-api
+    cp $src/app.py $out/libexec/stable-video-diffusion-api/app.py
+    chmod +x $out/libexec/stable-video-diffusion-api/app.py
+
+    makeWrapper ${pythonEnv}/bin/python3 $out/bin/stable-video-diffusion-api \
+      --add-flags $out/libexec/stable-video-diffusion-api/app.py \
+      --set-default SVD_API_HOST 127.0.0.1 \
+      --set-default SVD_API_PORT 8000 \
+      --set-default SVD_MODEL_ID stabilityai/stable-video-diffusion-img2vid-xt \
+      --set-default SVD_OUTPUT_DIR /var/lib/stable-video-diffusion-api/outputs
+  '';
+}