From 1a3e88c19ab568d97b2c3d98c7e968744c372adb Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 15:13:04 +0200 Subject: [PATCH 1/7] Ensure that the the jwt service is called before starting a call --- playwright/restricted-sfu.spec.ts | 75 ++++++++++++++++ playwright/sfu-reconnect-bug.spec.ts | 2 +- src/rtcSessionHelpers.ts | 125 +++++++++++++++++++++------ 3 files changed, 175 insertions(+), 27 deletions(-) create mode 100644 playwright/restricted-sfu.spec.ts diff --git a/playwright/restricted-sfu.spec.ts b/playwright/restricted-sfu.spec.ts new file mode 100644 index 00000000..a9e07d38 --- /dev/null +++ b/playwright/restricted-sfu.spec.ts @@ -0,0 +1,75 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; +import { sleep } from "matrix-js-sdk/lib/utils.js"; + +test("Should request JWT token before starting the call", async ({ page }) => { + await page.goto("/"); + + let sfGetTimestamp = 0; + let sendStateEventTimestamp = 0; + await page.route( + "**/matrix-rtc.m.localhost/livekit/jwt/sfu/get", + async (route) => { + await sleep(2000); // Simulate very slow request + await route.continue(); + sfGetTimestamp = Date.now(); + }, + ); + + await page.route( + "**/state/org.matrix.msc3401.call.member/**", + async (route) => { + await route.continue(); + sendStateEventTimestamp = Date.now(); + }, + ); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + await page.waitForTimeout(4000); + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + + expect(sfGetTimestamp).toBeGreaterThan(0); + expect(sendStateEventTimestamp).toBeGreaterThan(0); + expect(sfGetTimestamp).toBeLessThan(sendStateEventTimestamp); +}); + +test("Error when pre-warming the focus are caught by the ErrorBoundary", async ({ + page, +}) => { + await page.goto("/"); + + await page.route("**/openid/request_token", async (route) => { + await route.fulfill({ + status: 418, // Simulate an error not retryable + }); + }); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Should fail + await expect(page.getByText("Something went wrong")).toBeVisible(); +}); diff --git a/playwright/sfu-reconnect-bug.spec.ts b/playwright/sfu-reconnect-bug.spec.ts index c756570a..6138eb78 100644 --- a/playwright/sfu-reconnect-bug.spec.ts +++ b/playwright/sfu-reconnect-bug.spec.ts @@ -100,5 +100,5 @@ test("When creator left, avoid reconnect to the same SFU", async ({ // https://github.com/element-hq/element-call/issues/3344 // The app used to request a new jwt token then to reconnect to the SFU expect(wsConnectionCount).toBe(1); - expect(sfuGetCallCount).toBe(1); + expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index be5eedf1..3e2ff9cd 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -5,14 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; import { isLivekitFocus, isLivekitFocusConfig, type LivekitFocus, type LivekitFocusActive, + type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; @@ -20,6 +20,7 @@ import { Config } from "./config/Config"; import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; import { MatrixRTCFocusMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; +import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; @@ -51,32 +52,13 @@ async function makePreferredLivekitFoci( if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started - const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ - FOCI_WK_KEY - ]; - if (Array.isArray(wellKnownFoci)) { - preferredFoci.push( - ...wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - logger.log( - "Adding livekit focus from well known: ", - wellKnownFocus, - ); - return { ...wellKnownFocus, livekit_alias: livekitAlias }; - }), - ); - } + const wellKnownFoci = await getFocusListFromWellKnown(domain, livekitAlias); + logger.log("Adding livekit focus from well known: ", wellKnownFoci); + preferredFoci.push(...wellKnownFoci); } - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - const focusFormConf: LivekitFocus = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; + const focusFormConf = getFocusListFromConfig(livekitAlias); + if (focusFormConf) { logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } @@ -94,6 +76,93 @@ async function makePreferredLivekitFoci( // if (focusOtherMembers) preferredFoci.push(focusOtherMembers); } +async function getFocusListFromWellKnown( + domain: string, + alias: string, +): Promise { + if (domain) { + // we use AutoDiscovery instead of relying on the MatrixClient having already + // been fully configured and started + const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ + FOCI_WK_KEY + ]; + if (Array.isArray(wellKnownFoci)) { + return wellKnownFoci + .filter((f) => !!f) + .filter(isLivekitFocusConfig) + .map((wellKnownFocus) => { + return { ...wellKnownFocus, livekit_alias: alias }; + }); + } + } + return []; +} + +function getFocusListFromConfig(livekitAlias: string): LivekitFocus | null { + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + return { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + } + return null; +} + +export async function getMyPreferredLivekitFoci( + domain: string | null, + livekitAlias: string, +): Promise { + if (domain) { + // we use AutoDiscovery instead of relying on the MatrixClient having already + // been fully configured and started + const wellKnownFociList = await getFocusListFromWellKnown( + domain, + livekitAlias, + ); + if (wellKnownFociList.length > 0) { + return wellKnownFociList[0]; + } + } + + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + return { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + } + throw new MatrixRTCFocusMissingError(domain ?? ""); +} + +// Stop-gap solution for pre-warming the SFU. +// This is needed to ensure that the livekit room is created before we try to join the rtc session. +// This is because the livekit room creation is done by the auth service and this can be restricted to +// only specific users, so we need to ensure that the room is created before we try to join it. +async function preWarmSFU(rtcSession: MatrixRTCSession, livekitAlias: string) { + const client = rtcSession.room.client; + // We need to make sure that the livekit room is created before sending the membership event + // because other joiners might not be able to join the call if the room does not exist yet. + const fociToWarmup = await getMyPreferredLivekitFoci( + client.getDomain(), + livekitAlias + ); + + // Request a token in advance to warm up the livekit room. + // Let it throw if it fails, errors will be handled by the ErrorBoundary, if it fails now + // it will fail later when we try to join the room. + await getSFUConfigWithOpenID(client, fociToWarmup); + // For now we don't do anything with the token returned by `getSFUConfigWithOpenID`, it is just to ensure that we + // call the `sfu/get` endpoint so that the auth service create the room in advance if it can. + // Note: This is not actually checking that the room was created! If the roon creation is + // not done by the auth service, the call will fail later when we try to join the room; that case + // is a miss-configuration of the auth service, you should be able to create room in your selected SFU. + // A solution could be to call the internal `/validate` endpoint to check that the room exists, but this needs + // to access livekit internal APIs, so we don't do it for now. +} + export async function enterRTCSession( rtcSession: MatrixRTCSession, encryptMedia: boolean, @@ -112,6 +181,10 @@ export async function enterRTCSession( const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; + + // Pre-warm the SFU to ensure that the room is created before anyone tries to join it. + await preWarmSFU(rtcSession, livekitAlias); + rtcSession.joinRoomSession( await makePreferredLivekitFoci(rtcSession, livekitAlias), makeActiveFocus(), From dba8a434958d236bf4b6f83ce253c560a2ec4dee Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 15:30:15 +0200 Subject: [PATCH 2/7] fixup test mock --- src/rtcSessionHelpers.test.ts | 12 ++++++++++++ src/rtcSessionHelpers.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index ecfd44f7..2ef9e3f1 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -70,6 +70,12 @@ test("It joins the correct Session", async () => { roomId: "roomId", client: { getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), }, }, memberships: [], @@ -195,6 +201,12 @@ test("It should not fail with configuration error if homeserver config has livek roomId: "roomId", client: { getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), }, }, memberships: [], diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 3e2ff9cd..e1d97db9 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -147,7 +147,7 @@ async function preWarmSFU(rtcSession: MatrixRTCSession, livekitAlias: string) { // because other joiners might not be able to join the call if the room does not exist yet. const fociToWarmup = await getMyPreferredLivekitFoci( client.getDomain(), - livekitAlias + livekitAlias, ); // Request a token in advance to warm up the livekit room. From ef7c9a166a5d49fc6ab131485972d2ebcd651b04 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 15:58:46 +0200 Subject: [PATCH 3/7] fix eslint --- src/rtcSessionHelpers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index e1d97db9..75c1ff60 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -141,7 +141,10 @@ export async function getMyPreferredLivekitFoci( // This is needed to ensure that the livekit room is created before we try to join the rtc session. // This is because the livekit room creation is done by the auth service and this can be restricted to // only specific users, so we need to ensure that the room is created before we try to join it. -async function preWarmSFU(rtcSession: MatrixRTCSession, livekitAlias: string) { +async function preWarmSFU( + rtcSession: MatrixRTCSession, + livekitAlias: string, +): Promise { const client = rtcSession.room.client; // We need to make sure that the livekit room is created before sending the membership event // because other joiners might not be able to join the call if the room does not exist yet. From fce7b6d4569165d483d48c2e7896034d099c1ff6 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 Aug 2025 17:44:22 +0200 Subject: [PATCH 4/7] refactor: move warmup to makePreferredLivekitFoci --- src/rtcSessionHelpers.ts | 133 ++++++++++----------------------------- 1 file changed, 34 insertions(+), 99 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 75c1ff60..33150b48 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -47,18 +47,47 @@ async function makePreferredLivekitFoci( preferredFoci.push(focusInUse); } + // Stop-gap solution for pre-warming the SFU. + // This is needed to ensure that the livekit room is created before we try to join the rtc session. + // This is because the livekit room creation is done by the auth service and this can be restricted to + // only specific users, so we need to ensure that the room is created before we try to send state events. + let shouldWarmup = true; + // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started - const wellKnownFoci = await getFocusListFromWellKnown(domain, livekitAlias); - logger.log("Adding livekit focus from well known: ", wellKnownFoci); - preferredFoci.push(...wellKnownFoci); + const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ + FOCI_WK_KEY + ]; + if (Array.isArray(wellKnownFoci)) { + const validWellKnownFoci = wellKnownFoci + .filter((f) => !!f) + .filter(isLivekitFocusConfig) + .map((wellKnownFocus) => { + logger.log("Adding livekit focus from well known: ", wellKnownFocus); + return { ...wellKnownFocus, livekit_alias: livekitAlias }; + }); + if (validWellKnownFoci.length > 0) { + const toWarmup = validWellKnownFoci[0]; + await getSFUConfigWithOpenID(rtcSession.room.client, toWarmup); + shouldWarmup = false; + } + preferredFoci.push(...validWellKnownFoci); + } } - const focusFormConf = getFocusListFromConfig(livekitAlias); - if (focusFormConf) { + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + const focusFormConf: LivekitFocus = { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + if (shouldWarmup) { + await getSFUConfigWithOpenID(rtcSession.room.client, focusFormConf); + } logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } @@ -76,96 +105,6 @@ async function makePreferredLivekitFoci( // if (focusOtherMembers) preferredFoci.push(focusOtherMembers); } -async function getFocusListFromWellKnown( - domain: string, - alias: string, -): Promise { - if (domain) { - // we use AutoDiscovery instead of relying on the MatrixClient having already - // been fully configured and started - const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ - FOCI_WK_KEY - ]; - if (Array.isArray(wellKnownFoci)) { - return wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - return { ...wellKnownFocus, livekit_alias: alias }; - }); - } - } - return []; -} - -function getFocusListFromConfig(livekitAlias: string): LivekitFocus | null { - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - return { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - } - return null; -} - -export async function getMyPreferredLivekitFoci( - domain: string | null, - livekitAlias: string, -): Promise { - if (domain) { - // we use AutoDiscovery instead of relying on the MatrixClient having already - // been fully configured and started - const wellKnownFociList = await getFocusListFromWellKnown( - domain, - livekitAlias, - ); - if (wellKnownFociList.length > 0) { - return wellKnownFociList[0]; - } - } - - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - return { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - } - throw new MatrixRTCFocusMissingError(domain ?? ""); -} - -// Stop-gap solution for pre-warming the SFU. -// This is needed to ensure that the livekit room is created before we try to join the rtc session. -// This is because the livekit room creation is done by the auth service and this can be restricted to -// only specific users, so we need to ensure that the room is created before we try to join it. -async function preWarmSFU( - rtcSession: MatrixRTCSession, - livekitAlias: string, -): Promise { - const client = rtcSession.room.client; - // We need to make sure that the livekit room is created before sending the membership event - // because other joiners might not be able to join the call if the room does not exist yet. - const fociToWarmup = await getMyPreferredLivekitFoci( - client.getDomain(), - livekitAlias, - ); - - // Request a token in advance to warm up the livekit room. - // Let it throw if it fails, errors will be handled by the ErrorBoundary, if it fails now - // it will fail later when we try to join the room. - await getSFUConfigWithOpenID(client, fociToWarmup); - // For now we don't do anything with the token returned by `getSFUConfigWithOpenID`, it is just to ensure that we - // call the `sfu/get` endpoint so that the auth service create the room in advance if it can. - // Note: This is not actually checking that the room was created! If the roon creation is - // not done by the auth service, the call will fail later when we try to join the room; that case - // is a miss-configuration of the auth service, you should be able to create room in your selected SFU. - // A solution could be to call the internal `/validate` endpoint to check that the room exists, but this needs - // to access livekit internal APIs, so we don't do it for now. -} - export async function enterRTCSession( rtcSession: MatrixRTCSession, encryptMedia: boolean, @@ -184,10 +123,6 @@ export async function enterRTCSession( const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; - - // Pre-warm the SFU to ensure that the room is created before anyone tries to join it. - await preWarmSFU(rtcSession, livekitAlias); - rtcSession.joinRoomSession( await makePreferredLivekitFoci(rtcSession, livekitAlias), makeActiveFocus(), From 6dcc44b631abf3346db12daa81dcb1871febf630 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 6 Aug 2025 09:26:07 +0200 Subject: [PATCH 5/7] comment --- src/rtcSessionHelpers.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 33150b48..ba0b39f4 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -71,6 +71,7 @@ async function makePreferredLivekitFoci( }); if (validWellKnownFoci.length > 0) { const toWarmup = validWellKnownFoci[0]; + // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID(rtcSession.room.client, toWarmup); shouldWarmup = false; } @@ -86,6 +87,7 @@ async function makePreferredLivekitFoci( livekit_alias: livekitAlias, }; if (shouldWarmup) { + // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID(rtcSession.room.client, focusFormConf); } logger.log("Adding livekit focus from config: ", focusFormConf); From 8509efb48bcee81075d97c41aa47130faee35038 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 6 Aug 2025 13:59:53 +0200 Subject: [PATCH 6/7] review --- src/rtcSessionHelpers.ts | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index ba0b39f4..8fe45821 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -47,11 +47,8 @@ async function makePreferredLivekitFoci( preferredFoci.push(focusInUse); } - // Stop-gap solution for pre-warming the SFU. - // This is needed to ensure that the livekit room is created before we try to join the rtc session. - // This is because the livekit room creation is done by the auth service and this can be restricted to - // only specific users, so we need to ensure that the room is created before we try to send state events. - let shouldWarmup = true; + // Warm up the first focus we owned, to ensure livekit room is created before any state event sent. + let toWarmUp: LivekitFocus | undefined; // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); @@ -70,10 +67,7 @@ async function makePreferredLivekitFoci( return { ...wellKnownFocus, livekit_alias: livekitAlias }; }); if (validWellKnownFoci.length > 0) { - const toWarmup = validWellKnownFoci[0]; - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, toWarmup); - shouldWarmup = false; + toWarmUp = validWellKnownFoci[0]; } preferredFoci.push(...validWellKnownFoci); } @@ -86,14 +80,17 @@ async function makePreferredLivekitFoci( livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - if (shouldWarmup) { - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, focusFormConf); + if (!toWarmUp) { + toWarmUp = focusFormConf; } logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } + if (toWarmUp) { + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + await getSFUConfigWithOpenID(rtcSession.room.client, toWarmUp); + } if (preferredFoci.length === 0) throw new MatrixRTCFocusMissingError(domain ?? ""); return Promise.resolve(preferredFoci); From e4d14b4892be72327bfadf32d0f77a62e7cecd6c Mon Sep 17 00:00:00 2001 From: Valere Fedronic Date: Wed, 6 Aug 2025 14:22:10 +0200 Subject: [PATCH 7/7] Update src/rtcSessionHelpers.ts Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> --- src/rtcSessionHelpers.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 8fe45821..73f58cea 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -80,9 +80,7 @@ async function makePreferredLivekitFoci( livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - if (!toWarmUp) { - toWarmUp = focusFormConf; - } + toWarmUp = toWarmUp ?? focusFormConf; logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); }