[network/ruqola] src/core/encryption: Fix encoding session key. Now we can create new encrypted channel

Git commit 62bcbc30b1968f7d0bc74113829c5f9b09217b87 by Laurent Montel.
Committed on 06/08/2026 at 17:54.
Pushed by mlaurent into branch 'master'.

Fix encoding session key. Now we can create new encrypted channel

M  +17   -2    src/core/encryption/e2ekeymanager.cpp
M  +27   -0    src/core/encryption/encryptionutils.cpp
M  +1    -0    src/core/encryption/encryptionutils.h

https://invent.kde.org/network/ruqola/-/commit/62bcbc30b1968f7d0bc74113829c5f9b09217b87

diff --git a/src/core/encryption/e2ekeymanager.cpp b/src/core/encryption/e2ekeymanager.cpp
index 2d6be74ee6..43bc375a52 100644
--- a/src/core/encryption/e2ekeymanager.cpp
+++ b/src/core/encryption/e2ekeymanager.cpp
@@ -403,7 +403,15 @@ void E2eKeyManager::distributeRoomSessionKey([[maybe_unused]] const QByteArray &
         qCWarning(RUQOLA_ENCRYPTION_LOG) << "initializeRoomE2EKey: failed to parse own public key";
         return;
     }
-    const QByteArray encryptedSessionKey = EncryptionUtils::encryptSessionKey(sessionKey, rsaPublicKey);
+    // Rocket.Chat expects the RSA-encrypted payload to be the JWK JSON bytes of
+    // the session key (not raw bytes).  Wrap before encrypting.
+    const QByteArray sessionKeyJwk = EncryptionUtils::sessionKeyToJWK(sessionKey);
+    if (sessionKeyJwk.isEmpty()) {
+        RSA_free(rsaPublicKey);
+        qCWarning(RUQOLA_ENCRYPTION_LOG) << "initializeRoomE2EKey: failed to encode session key as JWK";
+        return;
+    }
+    const QByteArray encryptedSessionKey = EncryptionUtils::encryptSessionKey(sessionKeyJwk, rsaPublicKey);
     RSA_free(rsaPublicKey);
     if (encryptedSessionKey.isEmpty()) {
         qCWarning(RUQOLA_ENCRYPTION_LOG) << "initializeRoomE2EKey: session key encryption failed";
@@ -477,7 +485,14 @@ void E2eKeyManager::distributeRoomSessionKey([[maybe_unused]] const QByteArray &
                         continue;
                     }
 
-                    const QByteArray encryptedRecipientSessionKey = EncryptionUtils::encryptSessionKey(sessionKey, targetRsaPublicKey);
+                    // Encode session key as JWK before RSA-encrypting (Rocket.Chat format).
+                    const QByteArray recipientSessionKeyJwk = EncryptionUtils::sessionKeyToJWK(sessionKey);
+                    if (recipientSessionKeyJwk.isEmpty()) {
+                        RSA_free(targetRsaPublicKey);
+                        qCWarning(RUQOLA_ENCRYPTION_LOG) << "initializeRoomE2EKey: unable to encode session key as JWK for" << targetUserId;
+                        continue;
+                    }
+                    const QByteArray encryptedRecipientSessionKey = EncryptionUtils::encryptSessionKey(recipientSessionKeyJwk, targetRsaPublicKey);
                     RSA_free(targetRsaPublicKey);
                     if (encryptedRecipientSessionKey.isEmpty()) {
                         qCWarning(RUQOLA_ENCRYPTION_LOG) << "initializeRoomE2EKey: unable to encrypt room key for" << targetUserId;
diff --git a/src/core/encryption/encryptionutils.cpp b/src/core/encryption/encryptionutils.cpp
index 6a124e3657..0289a8f910 100644
--- a/src/core/encryption/encryptionutils.cpp
+++ b/src/core/encryption/encryptionutils.cpp
@@ -281,6 +281,33 @@ QByteArray EncryptionUtils::generateSessionKey()
     return generateRandomIV(32);
 }
 
+/**
+ * @brief Converts a raw 32-byte AES-256-GCM session key to JWK JSON format.
+ *
+ * Rocket.Chat distributes session keys as the RSA-OAEP-encrypted bytes of a JWK
+ * JSON string (not raw key bytes). This function produces the JSON payload that
+ * must be encrypted before sharing with other participants so that both Ruqola
+ * and Rocket.Chat web/mobile clients can import it.
+ *
+ * @param rawKey The 32-byte raw AES key.
+ * @return JWK JSON bytes, e.g.
+ *   {"k":"<base64url>","alg":"A256GCM","ext":true,"key_ops":["encrypt","decrypt"],"kty":"oct"}
+ */
+QByteArray EncryptionUtils::sessionKeyToJWK(const QByteArray &rawKey)
+{
+    if (rawKey.size() != 32) {
+        qCWarning(RUQOLA_ENCRYPTION_LOG) << "sessionKeyToJWK: expected 32-byte key, got" << rawKey.size();
+        return {};
+    }
+    QJsonObject jwk;
+    jwk[QStringLiteral("k")] = QString::fromLatin1(rawKey.toBase64(QByteArray::Base64UrlEncoding | QByteArray::OmitTrailingEquals));
+    jwk[QStringLiteral("alg")] = QStringLiteral("A256GCM");
+    jwk[QStringLiteral("ext")] = true;
+    jwk[QStringLiteral("key_ops")] = QJsonArray() << QStringLiteral("encrypt") << QStringLiteral("decrypt");
+    jwk[QStringLiteral("kty")] = QStringLiteral("oct");
+    return QJsonDocument(jwk).toJson(QJsonDocument::Compact);
+}
+
 /**
  * @brief Generates a room-specific key identifier (keyId).
  *
diff --git a/src/core/encryption/encryptionutils.h b/src/core/encryption/encryptionutils.h
index a18f4fef03..e129ff0811 100644
--- a/src/core/encryption/encryptionutils.h
+++ b/src/core/encryption/encryptionutils.h
@@ -46,6 +46,7 @@ struct RSAKeyPair {
 [[nodiscard]] LIBRUQOLACORE_EXPORT QByteArray deriveKey(const QByteArray &salt, const QByteArray &baseKey, int iterations = 1000, int keyLength = 32);
 [[nodiscard]] LIBRUQOLACORE_EXPORT QByteArray generateRandomIV(int size);
 [[nodiscard]] LIBRUQOLACORE_EXPORT QByteArray generateSessionKey();
+[[nodiscard]] LIBRUQOLACORE_EXPORT QByteArray sessionKeyToJWK(const QByteArray &rawKey);
 [[nodiscard]] LIBRUQOLACORE_EXPORT QByteArray encryptSessionKey(const QByteArray &sessionKey, RSA *publicKey);
 [[nodiscard]] LIBRUQOLACORE_EXPORT QByteArray decryptSessionKey(const QByteArray &encryptedSessionKey, RSA *privateKey);
 // Caller owns the returned RSA object and must release it with RSA_free().