From e38bf7582d2974d1e4612d269f43f26a985b355e Mon Sep 17 00:00:00 2001 From: yukkop Date: Sat, 23 May 2026 11:46:58 +0000 Subject: [PATCH] feat: media-browser for matrix --- nixos/module/hectic/service/media-browser.nix | 99 ++++ nixos/system/hectic-lab/hectic-lab.nix | 9 + package/default.nix | 1 + package/media-browser/app.py | 551 ++++++++++++++++++ package/media-browser/default.nix | 20 + 5 files changed, 680 insertions(+) create mode 100644 nixos/module/hectic/service/media-browser.nix create mode 100644 package/media-browser/app.py create mode 100644 package/media-browser/default.nix diff --git a/nixos/module/hectic/service/media-browser.nix b/nixos/module/hectic/service/media-browser.nix new file mode 100644 index 0000000..b812e06 --- /dev/null +++ b/nixos/module/hectic/service/media-browser.nix @@ -0,0 +1,99 @@ +{ + inputs, + flake, + self, +}: { + pkgs, + lib, + config, + ... +}: let + cfg = config.hectic.services.media-browser; + + mediaBrowserApp = pkgs.hectic.media-browser; +in { + options.hectic.services.media-browser = { + enable = lib.mkEnableOption "Matrix media browser web app"; + + port = lib.mkOption { + type = lib.types.port; + default = 3000; + description = "Port to bind the media browser web server."; + }; + + mediaStorePath = lib.mkOption { + type = lib.types.str; + default = "/var/lib/matrix-synapse/media_store"; + description = "Path to Synapse media store."; + }; + + s3CredentialsFile = lib.mkOption { + type = lib.types.path; + description = "Path to S3 credentials file (ACCESS_KEY_ID=..., SECRET_ACCESS_KEY=...)."; + }; + + s3Bucket = lib.mkOption { + type = lib.types.str; + description = "S3 bucket name."; + }; + + s3Endpoint = lib.mkOption { + type = lib.types.str; + description = "S3 endpoint URL."; + }; + + s3Region = lib.mkOption { + type = lib.types.str; + default = "hel1"; + description = "S3 region name."; + }; + + s3Prefix = lib.mkOption { + type = lib.types.str; + default = ""; + description = "S3 object key prefix."; + }; + + dbName = lib.mkOption { + type = lib.types.str; + default = "matrix-synapse"; + description = "PostgreSQL database name."; + }; + + dbUser = lib.mkOption { + type = lib.types.str; + default = "matrix-synapse"; + description = "PostgreSQL database user."; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.media-browser = { + description = "Matrix Media Browser"; + after = [ "network.target" "postgresql.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "simple"; + User = "matrix-synapse"; + Group = "matrix-synapse"; + ExecStart = "${mediaBrowserApp}/bin/media-browser-wrapped"; + Restart = "on-failure"; + RestartSec = 5; + }; + environment = { + FLASK_ENV = "production"; + PORT = toString cfg.port; + MEDIA_STORE_PATH = cfg.mediaStorePath; + S3_BUCKET = cfg.s3Bucket; + S3_ENDPOINT = cfg.s3Endpoint; + S3_REGION = cfg.s3Region; + S3_PREFIX = cfg.s3Prefix; + DB_NAME = cfg.dbName; + DB_USER = cfg.dbUser; + DB_HOST = "/run/postgresql"; + DB_PORT = "5432"; + }; + serviceConfig.EnvironmentFile = cfg.s3CredentialsFile; + }; + }; +} diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index 92d10b7..2669d97 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -84,6 +84,15 @@ in { }; inherit matrixDomain; }; + + services.media-browser = { + enable = true; + port = 3001; + s3Bucket = "matrix-hectic-lab"; + s3Endpoint = "https://hel1.your-objectstorage.com"; + s3Region = "hel1"; + s3CredentialsFile = config.sops.secrets."matrix/object-storage/credentials".path; + }; }; # NOTE(yukkop): disk was provisioned by Hetzner rescue image, disko was never diff --git a/package/default.nix b/package/default.nix index f5ce741..78f4219 100644 --- a/package/default.nix +++ b/package/default.nix @@ -169,4 +169,5 @@ in { pg-15-ext-smtp-client = buildSmtpExt pkgs "15"; pg-15-ext-plhaskell = buildPlHaskellExt pkgs "15"; pg-15-ext-plsh = buildPlShExt pkgs "15"; + media-browser = pkgs.callPackage ./media-browser {}; } diff --git a/package/media-browser/app.py b/package/media-browser/app.py new file mode 100644 index 0000000..9081875 --- /dev/null +++ b/package/media-browser/app.py @@ -0,0 +1,551 @@ +#!/usr/bin/env python3 +"""Matrix Media Browser - Browse local and S3 media with cross-reference.""" + +import os +import sys +import json +import psycopg2 +import boto3 +from datetime import datetime +from pathlib import Path +from urllib.parse import quote +from flask import Flask, render_template_string, send_file, jsonify, request + +app = Flask(__name__) + +MEDIA_STORE_PATH = os.environ.get('MEDIA_STORE_PATH', '/var/lib/matrix-synapse/media_store') +S3_BUCKET = os.environ.get('S3_BUCKET', 'matrix-hectic-lab') +S3_ENDPOINT = os.environ.get('S3_ENDPOINT', 'https://hel1.your-objectstorage.com') +S3_REGION = os.environ.get('S3_REGION', 'hel1') +S3_PREFIX = os.environ.get('S3_PREFIX', '') +DB_NAME = os.environ.get('DB_NAME', 'matrix-synapse') +DB_USER = os.environ.get('DB_USER', 'matrix-synapse') +DB_HOST = os.environ.get('DB_HOST', '/run/postgresql') +DB_PORT = int(os.environ.get('DB_PORT', '5432')) + +def get_db_conn(): + return psycopg2.connect( + dbname=DB_NAME, + user=DB_USER, + host=DB_HOST, + port=DB_PORT + ) + +def get_s3_client(): + return boto3.client('s3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=os.environ.get('ACCESS_KEY_ID', ''), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY', ''), + region_name=S3_REGION + ) + +@app.route('/') +def index(): + return render_template_string(HTML_TEMPLATE) + +@app.route('/api/stats') +def api_stats(): + try: + local_count = 0 + local_size = 0 + for root, dirs, files in os.walk(MEDIA_STORE_PATH): + for f in files: + local_count += 1 + local_size += os.path.getsize(os.path.join(root, f)) + + s3 = get_s3_client() + s3_count = 0 + s3_size = 0 + paginator = s3.get_paginator('list_objects_v2') + list_kwargs = {'Bucket': S3_BUCKET} + if S3_PREFIX: + list_kwargs['Prefix'] = S3_PREFIX + '/' + for page in paginator.paginate(**list_kwargs): + for obj in page.get('Contents', []): + s3_count += 1 + s3_size += obj['Size'] + + conn = get_db_conn() + cur = conn.cursor() + cur.execute("SELECT COUNT(*), COALESCE(SUM(media_length), 0) FROM local_media_repository") + db_count, db_size = cur.fetchone() + cur.execute("SELECT COUNT(*) FROM remote_media_cache") + remote_count = cur.fetchone()[0] + cur.close() + conn.close() + + return jsonify({ + 'local_files': local_count, + 'local_size': local_size, + 's3_objects': s3_count, + 's3_size': s3_size, + 'db_local_entries': db_count or 0, + 'db_total_size': int(db_size) if db_size else 0, + 'db_remote_entries': remote_count or 0 + }) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/api/local') +def api_local(): + try: + files = [] + for root, dirs, filenames in os.walk(MEDIA_STORE_PATH): + for filename in filenames: + filepath = os.path.join(root, filename) + rel_path = os.path.relpath(filepath, MEDIA_STORE_PATH) + stat = os.stat(filepath) + files.append({ + 'path': rel_path, + 'size': stat.st_size, + 'modified': datetime.fromtimestamp(stat.st_mtime).isoformat(), + 'full_path': filepath + }) + return jsonify(files) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/api/s3') +def api_s3(): + try: + s3 = get_s3_client() + objects = [] + paginator = s3.get_paginator('list_objects_v2') + list_kwargs = {'Bucket': S3_BUCKET} + if S3_PREFIX: + list_kwargs['Prefix'] = S3_PREFIX + '/' + for page in paginator.paginate(**list_kwargs): + for obj in page.get('Contents', []): + objects.append({ + 'key': obj['Key'], + 'size': obj['Size'], + 'modified': obj['LastModified'].isoformat(), + 'etag': obj['ETag'].strip('"') + }) + return jsonify(objects) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/api/media-db') +def api_media_db(): + try: + conn = get_db_conn() + cur = conn.cursor() + + limit = request.args.get('limit', 1000, type=int) + offset = request.args.get('offset', 0, type=int) + + cur.execute(""" + SELECT media_id, media_type, media_length, created_ts, + last_access_ts, upload_name, quarantined_by + FROM local_media_repository + ORDER BY created_ts DESC + LIMIT %s OFFSET %s + """, (limit, offset)) + + rows = [] + for row in cur.fetchall(): + media_id, media_type, media_length, created_ts, last_access_ts, upload_name, quarantined = row + media_path = f"local_content/{media_id[0:2]}/{media_id[2:4]}/{media_id[4:]}" + local_exists = os.path.exists(os.path.join(MEDIA_STORE_PATH, media_path)) + + rows.append({ + 'media_id': media_id, + 'media_type': media_type, + 'size': media_length, + 'created': datetime.fromtimestamp(created_ts / 1000).isoformat() if created_ts else None, + 'last_access': datetime.fromtimestamp(last_access_ts / 1000).isoformat() if last_access_ts else None, + 'upload_name': upload_name, + 'quarantined': quarantined is not None, + 'local_path': media_path, + 'local_exists': local_exists + }) + + cur.close() + conn.close() + return jsonify(rows) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/api/sync-status') +def api_sync_status(): + try: + local_files = set() + for root, dirs, filenames in os.walk(MEDIA_STORE_PATH): + for filename in filenames: + filepath = os.path.join(root, filename) + rel_path = os.path.relpath(filepath, MEDIA_STORE_PATH) + local_files.add(rel_path) + + s3 = get_s3_client() + s3_files = set() + paginator = s3.get_paginator('list_objects_v2') + list_kwargs = {'Bucket': S3_BUCKET} + prefix = '' + if S3_PREFIX: + prefix = S3_PREFIX + '/' + list_kwargs['Prefix'] = prefix + for page in paginator.paginate(**list_kwargs): + for obj in page.get('Contents', []): + key = obj['Key'] + if prefix: + key = key[len(prefix):] + s3_files.add(key) + + synced = local_files & s3_files + local_only = local_files - s3_files + s3_only = s3_files - local_files + + return jsonify({ + 'synced_count': len(synced), + 'local_only_count': len(local_only), + 's3_only_count': len(s3_only), + 'synced': sorted(list(synced))[:100], + 'local_only': sorted(list(local_only))[:100], + 's3_only': sorted(list(s3_only))[:100] + }) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/view/local/') +def view_local(filepath): + try: + safe_path = os.path.join(MEDIA_STORE_PATH, filepath) + # Security: ensure path is within MEDIA_STORE_PATH + real_path = os.path.realpath(safe_path) + real_base = os.path.realpath(MEDIA_STORE_PATH) + if not real_path.startswith(real_base): + return 'Access denied', 403 + + if not os.path.exists(real_path): + return 'Not found', 404 + + return send_file(real_path) + except Exception as e: + return str(e), 500 + +@app.route('/view/s3/') +def view_s3(key): + try: + s3 = get_s3_client() + full_key = f"{S3_PREFIX}/{key}" if S3_PREFIX else key + + url = s3.generate_presigned_url('get_object', + Params={'Bucket': S3_BUCKET, 'Key': full_key}, + ExpiresIn=3600) + + return jsonify({'url': url}) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +HTML_TEMPLATE = ''' + + + + + Matrix Media Browser + + + +

📁 Matrix Media Browser

+ +
+
-
Local Files
+
-
S3 Objects
+
-
DB Entries
+
-
Synced
+
+ +
+ + + + +
+ +
+
+

Local Media Repository (from DB)

+
Loading...
+
+ + + +
+ + + + +''' + +if __name__ == '__main__': + port = int(os.environ.get('PORT', '3000')) + app.run(host='127.0.0.1', port=port, debug=False) diff --git a/package/media-browser/default.nix b/package/media-browser/default.nix new file mode 100644 index 0000000..77e2964 --- /dev/null +++ b/package/media-browser/default.nix @@ -0,0 +1,20 @@ +{ pkgs }: + +pkgs.stdenv.mkDerivation { + pname = "media-browser"; + version = "0.1.0"; + src = ./.; + nativeBuildInputs = [ pkgs.makeWrapper ]; + installPhase = '' + mkdir -p $out/bin + cp $src/app.py $out/bin/media-browser + chmod +x $out/bin/media-browser + makeWrapper ${pkgs.python3.withPackages (ps: [ + ps.flask + ps.psycopg2 + ps.boto3 + ps.pyyaml + ])}/bin/python3 $out/bin/media-browser-wrapped \ + --add-flags $out/bin/media-browser + ''; +}