From ef59f14372d2a711b9a0f7c4628359e39d0242de Mon Sep 17 00:00:00 2001
From: yukkop <hectic.yukkop@gmail.com>
Date: Sat, 23 May 2026 09:59:34 +0000
Subject: [PATCH] feat: `matrix`: s3 object storage

---
 nixos/module/hectic/service/matrix.nix | 501 ++++++++++++++++++-------
 nixos/system/hectic-lab/hectic-lab.nix |  12 +
 sus/hectic-lab.yaml                    |   6 +-
 3 files changed, 377 insertions(+), 142 deletions(-)

diff --git a/nixos/module/hectic/service/matrix.nix b/nixos/module/hectic/service/matrix.nix
index b507c80..c7d5dc8 100644
--- a/nixos/module/hectic/service/matrix.nix
+++ b/nixos/module/hectic/service/matrix.nix
@@ -10,7 +10,14 @@
   ...
 }: let
   cfg = config.hectic.services.matrix;
+  s3Cfg = cfg.objectStorage.s3;
+
   matrixUsers = builtins.attrNames cfg.users;
+
+  s3Plugin = pkgs.matrix-synapse-plugins.matrix-synapse-s3-storage-provider;
+  s3ConfigDir = "/run/matrix-synapse";
+  s3ConfigFile = "${s3ConfigDir}/s3-media-storage.yaml";
+
   mkUserRegistration = name: let
     user = cfg.users.${name};
     adminFlag = if user.admin then "--admin" else "--no-admin";
@@ -27,10 +34,58 @@
       ${adminFlag} \
       http://127.0.0.1:8008 || true
   '';
+
+  mkS3Config = ''
+    if [ ! -r "${s3Cfg.credentialsFile}" ]; then
+      printf 'Missing Matrix object storage credentials file: %s\n' '${s3Cfg.credentialsFile}' >&2
+      exit 1
+    fi
+
+    . "${s3Cfg.credentialsFile}"
+
+    if [ -z "$ACCESS_KEY_ID" ] || [ -z "$SECRET_ACCESS_KEY" ]; then
+      printf 'ACCESS_KEY_ID or SECRET_ACCESS_KEY missing in %s\n' '${s3Cfg.credentialsFile}' >&2
+      exit 1
+    fi
+
+    mkdir -p "${s3ConfigDir}"
+
+    cat > "${s3ConfigFile}" <<EOF
+    media_storage_providers:
+      - module: s3_storage_provider.S3StorageProviderBackend
+        store_local: ${lib.boolToString s3Cfg.storeLocal}
+        store_remote: ${lib.boolToString s3Cfg.storeRemote}
+        store_synchronous: ${lib.boolToString s3Cfg.storeSynchronous}
+        config:
+          bucket: ${s3Cfg.bucket}
+          endpoint_url: ${s3Cfg.endpointUrl}
+          region_name: ${s3Cfg.regionName}
+          prefix: "${s3Cfg.prefix}"
+          storage_class: "${s3Cfg.storageClass}"
+          threadpool_size: ${toString s3Cfg.threadpoolSize}
+          access_key_id: $ACCESS_KEY_ID
+          secret_access_key: $SECRET_ACCESS_KEY
+    EOF
+
+    chown matrix-synapse:matrix-synapse "${s3ConfigFile}"
+    chmod 0400 "${s3ConfigFile}"
+  '';
+
+  mkS3SyncScript = ''
+    ${s3Plugin}/bin/s3_media_upload write
+    ${s3Plugin}/bin/s3_media_upload upload "${s3Cfg.mediaStorePath}" "${s3Cfg.bucket}" \
+      --endpoint-url "${s3Cfg.endpointUrl}" \
+      --storage-class "${s3Cfg.storageClass}" \
+      ${lib.optionalString (s3Cfg.prefix != "") "--prefix \"${s3Cfg.prefix}\" \\"}
+      ${lib.optionalString s3Cfg.sync.deleteLocalAfterUpload "--delete"}
+    ${s3Plugin}/bin/s3_media_upload update-db 0s
+    ${s3Plugin}/bin/s3_media_upload check-deleted "${s3Cfg.mediaStorePath}"
+  '';
 in {
   options = {
     hectic.services.matrix = {
       enable = lib.mkEnableOption "Matrix Synapse homeserver with PostgreSQL and nginx";
+
       secretsFile = lib.mkOption {
         type = lib.types.path;
         description = ''
@@ -42,6 +97,7 @@ in {
           form_secret
         '';
       };
+
       postgresql = {
         port = lib.mkOption {
           type = lib.types.port;
@@ -50,6 +106,7 @@ in {
             postgres port
           '';
         };
+
         initialEnvFile = lib.mkOption {
           type = lib.types.path;
           description = ''
@@ -60,12 +117,14 @@ in {
           '';
         };
       };
+
       matrixDomain = lib.mkOption {
         type = lib.types.str;
         description = ''
           domain to matrix
         '';
       };
+
       maxUploadSize = lib.mkOption {
         type = lib.types.str;
         default = "100M";
@@ -73,6 +132,131 @@ in {
           Maximum file upload size accepted by Synapse and nginx.
         '';
       };
+
+      objectStorage.s3 = {
+        enable = lib.mkEnableOption "S3-compatible object storage for Matrix media";
+
+        bucket = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            Bucket name used for Matrix media objects.
+          '';
+        };
+
+        regionName = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            Region name passed to the Synapse S3 storage provider.
+          '';
+        };
+
+        endpointUrl = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            S3-compatible endpoint URL.
+          '';
+        };
+
+        credentialsFile = lib.mkOption {
+          type = lib.types.path;
+          description = ''
+            Path to an env-style file containing:
+              ACCESS_KEY_ID=
+              SECRET_ACCESS_KEY=
+          '';
+        };
+
+        mediaStorePath = lib.mkOption {
+          type = lib.types.str;
+          default = "/var/lib/matrix-synapse/media_store";
+          description = ''
+            Local Synapse media store path used before upload to object storage.
+          '';
+        };
+
+        prefix = lib.mkOption {
+          type = lib.types.str;
+          default = "";
+          description = ''
+            Optional object key prefix inside the bucket.
+          '';
+        };
+
+        storageClass = lib.mkOption {
+          type = lib.types.str;
+          default = "STANDARD";
+          description = ''
+            Storage class passed to the upload tool.
+          '';
+        };
+
+        threadpoolSize = lib.mkOption {
+          type = lib.types.int;
+          default = 40;
+          description = ''
+            Worker pool size for the Synapse S3 storage provider.
+          '';
+        };
+
+        storeLocal = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Mirror local uploads to object storage.
+          '';
+        };
+
+        storeRemote = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Mirror remotely-fetched media to object storage.
+          '';
+        };
+
+        storeSynchronous = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Wait for object storage upload before completing the client request.
+          '';
+        };
+
+        sync = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Periodically migrate older local media to object storage.
+            '';
+          };
+
+          olderThan = lib.mkOption {
+            type = lib.types.str;
+            default = "1d";
+            description = ''
+              Age threshold passed to `s3_media_upload update`.
+            '';
+          };
+
+          onCalendar = lib.mkOption {
+            type = lib.types.str;
+            default = "hourly";
+            description = ''
+              systemd timer schedule for media sync.
+            '';
+          };
+
+          deleteLocalAfterUpload = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Remove local media after successful object storage upload.
+            '';
+          };
+        };
+      };
+
       users = lib.mkOption {
         type = lib.types.attrsOf (lib.types.submodule {
           options = {
@@ -82,6 +266,7 @@ in {
                 Full path to a file containing the Matrix user's password.
               '';
             };
+
             admin = lib.mkOption {
               type = lib.types.bool;
               default = false;
@@ -98,166 +283,202 @@ in {
       };
     };
   };
-  config = lib.mkIf cfg.enable {
-    services.matrix-synapse = {
-      enable = true;
-      extraConfigFiles = [
-        cfg.secretsFile
-      ];
-      settings = {
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.enable {
+      services.matrix-synapse = {
+        enable = true;
+        plugins = lib.optional s3Cfg.enable s3Plugin;
+        extraConfigFiles = [
+          cfg.secretsFile
+        ] ++ lib.optional s3Cfg.enable s3ConfigFile;
+
+        settings = {
           server_name = cfg.matrixDomain;
           public_baseurl = "https://${cfg.matrixDomain}";
           max_upload_size = cfg.maxUploadSize;
+          media_store_path = lib.mkIf s3Cfg.enable s3Cfg.mediaStorePath;
+
           experimental_features = {
             msc3266_enabled = true;
             msc4140_enabled = true;
-           msc4143_enabled = true;
-           msc4222_enabled = true;
-         };
-         matrix_rtc = {
-           transports = [
-             {
-               type = "livekit";
-               livekit_service_url = "https://${cfg.matrixDomain}/livekit/jwt";
-             }
-           ];
-         };
-         listeners = [
-           {
-             port = 8008;
-             bind_addresses = [ "0.0.0.0" ];
-             type = "http";
-            tls = false;
-            resources = [
+            msc4143_enabled = true;
+            msc4222_enabled = true;
+          };
+
+          matrix_rtc = {
+            transports = [
               {
-                 names = [
-                   "client"
-                   # Ability speak between different matrix servers and get
-                   # global id, requires .well-known
-                   "federation"
-                   "openid"
-                 ];
-                 compress = false;
-               }
-             ];
+                type = "livekit";
+                livekit_service_url = "https://${cfg.matrixDomain}/livekit/jwt";
+              }
+            ];
+          };
+
+          listeners = [
+            {
+              port = 8008;
+              bind_addresses = [ "0.0.0.0" ];
+              type = "http";
+              tls = false;
+              resources = [
+                {
+                  names = [
+                    "client"
+                    # Ability speak between different matrix servers and get
+                    # global id, requires .well-known
+                    "federation"
+                    "openid"
+                  ];
+                  compress = false;
+                }
+              ];
+            }
+          ];
+
+          enable_registration = true;
+          enable_registration_without_verification = true;
+        };
+      };
+
+      environment.systemPackages = [
+        pkgs.matrix-synapse
+      ];
+
+      services.postgresql = {
+        enable = true;
+        package = pkgs.postgresql_17;
+
+        initdbArgs = [
+          "--locale=C"
+          "--encoding=UTF8"
+        ];
+
+        enableTCPIP = true;
+        settings.port = cfg.postgresql.port;
+        authentication = builtins.concatStringsSep "\n" [
+          "local all         all           trust"
+          "host  sameuser    all           127.0.0.1/32 scram-sha-256"
+          "host  sameuser    all           ::1/128 scram-sha-256"
+          "host  all         all           ::1/128 scram-sha-256"
+          "host  all         all           0.0.0.0/0 scram-sha-256"
+
+          "host  replication postgres      127.0.0.1/32   scram-sha-256"
+          "host  replication postgres      ::1/128        scram-sha-256"
+        ];
+
+        settings = {
+          wal_level = "replica";
+          max_wal_senders = 10;
+        };
+
+        ensureUsers = [
+          {
+            name = "matrix-synapse";
+            ensureClauses.login = true;
+            ensureDBOwnership = true;
           }
         ];
 
-        enable_registration = true;
-        enable_registration_without_verification = true;
+        ensureDatabases = [
+          "matrix-synapse"
+        ];
 
-      };
-    };
+        initialScript = pkgs.writeText "init-sql-script" ''
+          -- setup password from env/sops
+          DO $$#!${pkgs.dash}/bin/dash
+            set -e
+            . ${cfg.postgresql.initialEnvFile}
+            psql -Atc "ALTER USER postgres WITH PASSWORD '$POSTGRESQL_PASSWORD'";
+          $$ LANGUAGE plsh;
 
-    environment.systemPackages = [
-      pkgs.matrix-synapse
-    ];
-
-    services.postgresql = {
-      enable = true;
-      package = pkgs.postgresql_17;
-
-      initdbArgs = [
-        "--locale=C"
-        "--encoding=UTF8"
-      ];
-
-      enableTCPIP = true;
-      settings.port = cfg.postgresql.port;
-      authentication = builtins.concatStringsSep "\n" [
-        "local all         all           trust"
-        "host  sameuser    all           127.0.0.1/32 scram-sha-256"
-        "host  sameuser    all           ::1/128 scram-sha-256"
-        "host  all         all           ::1/128 scram-sha-256"
-        "host  all         all           0.0.0.0/0 scram-sha-256"
-
-        "host  replication postgres      127.0.0.1/32   scram-sha-256"
-        "host  replication postgres      ::1/128        scram-sha-256"
-      ];
-
-      settings = {
-        wal_level = "replica";
-        max_wal_senders = 10;
+          CREATE ROLE myuser LOGIN PASSWORD 'matrix-synapse';
+        '';
       };
 
-      ensureUsers = [
-        {
-          name = "matrix-synapse";
-          ensureClauses.login = true;
-          ensureDBOwnership = true;
-        }
-      ];
-
-      ensureDatabases = [
-        "matrix-synapse"
-      ];
-
-      initialScript = pkgs.writeText "init-sql-script" ''
-        -- setup password from env/sops
-        DO $$#!${pkgs.dash}/bin/dash
-          set -e
-          . ${cfg.postgresql.initialEnvFile}
-          psql -Atc "ALTER USER postgres WITH PASSWORD '$POSTGRESQL_PASSWORD'";
-        $$ LANGUAGE plsh;
-
-        CREATE ROLE myuser LOGIN PASSWORD 'matrix-synapse';
-      '';
-    };
-
-    services.nginx = {
-      enable = true;
-      virtualHosts.${cfg.matrixDomain} = {
-        forceSSL = true;
-        enableACME = true;
-        locations."/" = {
-          proxyPass = "http://127.0.0.1:8008";
-          extraConfig = ''
-            client_max_body_size ${cfg.maxUploadSize};
-          '';
-        };
-        locations."=/.well-known/matrix/server" = {
-          extraConfig = ''
-            default_type application/json;
-            add_header Access-Control-Allow-Origin *;
-            add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS";
-            add_header Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization";
-          '';
-          return = "200 '{\"m.server\": \"${cfg.matrixDomain}:443\"}'";
+      services.nginx = {
+        enable = true;
+        virtualHosts.${cfg.matrixDomain} = {
+          forceSSL = true;
+          enableACME = true;
+          locations."/" = {
+            proxyPass = "http://127.0.0.1:8008";
+            extraConfig = ''
+              client_max_body_size ${cfg.maxUploadSize};
+            '';
+          };
+          locations."=/.well-known/matrix/server" = {
+            extraConfig = ''
+              default_type application/json;
+              add_header Access-Control-Allow-Origin *;
+              add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS";
+              add_header Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization";
+            '';
+            return = "200 '{\"m.server\": \"${cfg.matrixDomain}:443\"}'";
+          };
         };
       };
-    };
 
-    security.acme = {
-      acceptTerms = true;
-      defaults = {
-        email = "hectic.yukkop.it@gmail.com";
-        enableDebugLogs = true;
+      security.acme = {
+        acceptTerms = true;
+        defaults = {
+          email = "hectic.yukkop.it@gmail.com";
+          enableDebugLogs = true;
+        };
       };
-    };
-    systemd.services.matrix-synapse-users = lib.mkIf (matrixUsers != []) {
-      description = "Provision Matrix Synapse users";
-      wantedBy = [ "multi-user.target" ];
-      after = [ config.services.matrix-synapse.serviceUnit ];
-      requires = [ config.services.matrix-synapse.serviceUnit ];
-      path = with pkgs; [ curl coreutils gawk ];
-      serviceConfig = {
-        Type = "oneshot";
-        User = "matrix-synapse";
-      };
-      script = ''
-        until curl -sf http://127.0.0.1:8008/_matrix/client/versions >/dev/null; do
-          sleep 2
-        done
 
-        REGISTRATION_SHARED_SECRET="$(awk -F': *' '$1 == "registration_shared_secret" { print $2; exit }' "${cfg.secretsFile}")"
+      systemd.services.matrix-synapse-users = lib.mkIf (matrixUsers != []) {
+        description = "Provision Matrix Synapse users";
+        wantedBy = [ "multi-user.target" ];
+        after = [ config.services.matrix-synapse.serviceUnit ];
+        requires = [ config.services.matrix-synapse.serviceUnit ];
+        path = with pkgs; [ curl coreutils gawk ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "matrix-synapse";
+        };
+        script = ''
+          until curl -sf http://127.0.0.1:8008/_matrix/client/versions >/dev/null; do
+            sleep 2
+          done
 
-        if [ -z "$REGISTRATION_SHARED_SECRET" ]; then
-          printf 'registration_shared_secret not found in %s\n' '${cfg.secretsFile}' >&2
-          exit 1
-        fi
+          REGISTRATION_SHARED_SECRET="$(awk -F': *' '$1 == "registration_shared_secret" { print $2; exit }' "${cfg.secretsFile}")"
+
+          if [ -z "$REGISTRATION_SHARED_SECRET" ]; then
+            printf 'registration_shared_secret not found in %s\n' '${cfg.secretsFile}' >&2
+            exit 1
+          fi
 
 ${builtins.concatStringsSep "\n" (map mkUserRegistration matrixUsers)}
-      '';
-    };
-  };
+        '';
+      };
+    })
+
+    (lib.mkIf (cfg.enable && s3Cfg.enable) {
+      systemd.services.matrix-synapse-s3-config = {
+        description = "Generate Synapse S3 media storage config";
+        before = [ config.services.matrix-synapse.serviceUnit ];
+        requiredBy = [ config.services.matrix-synapse.serviceUnit ];
+        serviceConfig.Type = "oneshot";
+        script = mkS3Config;
+      };
+
+      systemd.services.matrix-synapse-s3-media-sync = lib.mkIf s3Cfg.sync.enable {
+        description = "Sync Matrix media to S3-compatible object storage";
+        after = [ config.services.matrix-synapse.serviceUnit ];
+        wants = [ config.services.matrix-synapse.serviceUnit ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "matrix-synapse";
+          WorkingDirectory = "/var/lib/matrix-synapse";
+        };
+        script = mkS3SyncScript;
+      };
+
+      systemd.timers.matrix-synapse-s3-media-sync = lib.mkIf s3Cfg.sync.enable {
+        wantedBy = [ "timers.target" ];
+        timerConfig.OnCalendar = s3Cfg.sync.onCalendar;
+      };
+    })
+  ];
 }
diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix
index da52c9a..92d10b7 100644
--- a/nixos/system/hectic-lab/hectic-lab.nix
+++ b/nixos/system/hectic-lab/hectic-lab.nix
@@ -75,6 +75,13 @@ in {
           passwordFile = config.sops.secrets."matrix/users/lvgkcfjl/password".path;
         };
       };
+      objectStorage.s3 = {
+        enable = true;
+        bucket = "matrix-hectic-lab";
+        regionName = "hel1";
+        endpointUrl = "https://hel1.your-objectstorage.com";
+        credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
+      };
       inherit matrixDomain;
     };
   };
@@ -169,6 +176,11 @@ in {
     key = "matrix/users/lvgkcfjl/password";
     owner = "matrix-synapse";
   };
+  sops.secrets."matrix/object-storage/credentials" = {
+    key = "matrix/object-storage/credentials";
+    owner = "matrix-synapse";
+    mode = "0400";
+  };
 
   services.mailserver = {
     enable = true;
diff --git a/sus/hectic-lab.yaml b/sus/hectic-lab.yaml
index 02979ed..5adcaff 100644
--- a/sus/hectic-lab.yaml
+++ b/sus/hectic-lab.yaml
@@ -13,6 +13,8 @@ mailserver:
         hashedPassword: ENC[AES256_GCM,data:6Rgj4JIrEF9ZRRRwGpV4yCdS7cw81xKLfavuii1cHqZK3JDlD2HOAVYgrrl+fWD6rNxUPAXpVuAIgxCu,iv:Y67je0qtEpnbwhiYXL2FJUAedPlKdTTb6wGeSVVEaPQ=,tag:Thvt+gsebEjoIjwOmNgBGQ==,type:str]
 init-postgresql: ENC[AES256_GCM,data:Iw8M2P1QoqPVaEdM8Zo0qlHrYgop0iknDY4NtgDo,iv:RWj9AFnh4/KWCm3UH4RoCdM2lzsXGY7A7qko8xCxjp8=,tag:l8acSq8+NBXB4L1rVzG6kw==,type:str]
 matrix:
+    object-storage:
+        credentials: ENC[AES256_GCM,data:n2sDhGMR8y0in9pdn4zNEQBC5dqk+4JwbuJgEeQxyjn8bL9GebFBaqeE+frvPAGXj/DgpU6lFlFPgaGTWaMZAEEVpXyFeOdODpgW049q83ug5e4j/mbZgFM36XoItw==,iv:MW9H0zASdrY7SX1XM/jfoBihBYX0Fmlew4f71AvvV6Y=,tag:cAiOKUtOeTnczudps8YgQw==,type:str]
     secrets: ENC[AES256_GCM,data:ivXp2YSiMI4hgL6122Ex+fGW0lsZvGD6XmiRvNgFgvzLH5yDv9uLsYcGCTYfQSL3X5VyIMGvsdRF+4pbIjBZMuQKrjvXv74E7aFBLQ2Qk98N3IIrznUFR3KXbHR6xXy5ILd7Bmw5JI/ZHULbmITahXUBt2kEJvfh4eAtqShNA4vsJrabHX9A8Q+2Ddp16w0cWftV5++WXzlNpvIc2Py6BwvfroNAjpSaO+ILYDOIL7XjPvF83fTt64pxZ9nsi3hCzcDtBgGkqc8=,iv:wvt9V2uYQUwivSwEIYZwcHjXr5WwMw19lgFDIa1CcVw=,tag:/22UZvp7+1hLbt+kV+wokQ==,type:str]
     users:
         yukkop:
@@ -83,7 +85,7 @@ sops:
             Yk43ZmlTc09aNFV1VjdjN2RWQlFWTDQKcYSvA2lHP8GS0lkYY19Tm8RXmFHQX5Ck
             qV2Fn22Fic4M5FVKDEMfaO6WmeXgki9a8dGeO9LlC+Phf16SOq7eLw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-23T07:08:25Z"
-    mac: ENC[AES256_GCM,data:D7V2jX6AK89s/TRjrNXBHHyJHIbp+OQSzV2XfZ3qUOxJKfrNvNes8tARGO7fF1OMdbnZJVC7VBCMVOg0UtN6UlLepF9lL/jYzKPbPO6ohhhTdgv4OUkiTNVZ6MGQOY2win9NiHoAhn6zdNw8bZeXNPN5D7eY+Spy6zXLvjrl2EY=,iv:CoVVHHmFga/ecqP0KNp0Gy6rx08SuqImNuc6zc0JGpU=,tag:s4tJBGtBUjRyDCkPXqaQAQ==,type:str]
+    lastmodified: "2026-05-23T09:54:16Z"
+    mac: ENC[AES256_GCM,data:n3ljtQmDWFbhYo8eXjxPpE6xi/HIAMHaT1SATANIHi2Ged0c5PI/YS8jodRPUdb4zgLSMtoJjX5bqfJZ1en4WBY1rg69hDXst2sZ42zmDkafGTFpPBoS+QtC1c9UYQwvJfJ+4fX5qIcO6wplhIZG5PZrqb2dpPUD4vkRNxez2ho=,iv:ny5RCC7/3bNTiR2V4Hn3DuLdn8lThZeUKoCwXrzL0V0=,tag:v6THc2C4w7Uujh1VLKcokw==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.10.2