From 9011ae4e1f9a0f28f336adf0b11ec9b38c81d158 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 14:01:01 +0200 Subject: [PATCH 001/144] temp Signed-off-by: Timo K --- src/livekit/MatrixAudioRenderer.tsx | 2 + src/livekit/livekitSubscriptionRoom.ts | 123 +++++++++++ src/livekit/openIDSFU.ts | 66 ++---- src/livekit/useLivekit.ts | 49 +++-- src/room/InCallView.tsx | 133 ++++-------- src/rtcSessionHelpers.ts | 117 ++++------ src/state/CallViewModel.ts | 283 ++++++++++++++++++------- yarn.lock | 2 +- 8 files changed, 461 insertions(+), 314 deletions(-) create mode 100644 src/livekit/livekitSubscriptionRoom.ts diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 24975509..4b28a733 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -78,6 +78,8 @@ export function MatrixAudioRenderer({ loggedInvalidIdentities.current.add(identity); }; + // TODO-MULTI-SFU this uses the livekit room form the context. We need to change it so it uses the + // livekit room explicitly so we can pass a list of rooms into the audio renderer and call useTracks for each room. const tracks = useTracks( [ Track.Source.Microphone, diff --git a/src/livekit/livekitSubscriptionRoom.ts b/src/livekit/livekitSubscriptionRoom.ts new file mode 100644 index 00000000..f92ff10e --- /dev/null +++ b/src/livekit/livekitSubscriptionRoom.ts @@ -0,0 +1,123 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + ConnectionState, + type E2EEManagerOptions, + ExternalE2EEKeyProvider, + LocalVideoTrack, + Room, + type RoomOptions, +} from "livekit-client"; +import { useEffect, useRef } from "react"; +import E2EEWorker from "livekit-client/e2ee-worker?worker"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; + +import { defaultLiveKitOptions } from "./options"; +import { type SFUConfig } from "./openIDSFU"; +import { type MuteStates } from "../room/MuteStates"; +import { useMediaDevices } from "../MediaDevicesContext"; +import { + type ECConnectionState, + useECConnectionState, +} from "./useECConnectionState"; +import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; +import { E2eeType } from "../e2ee/e2eeType"; +import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; +import { + useTrackProcessor, + useTrackProcessorSync, +} from "./TrackProcessorContext"; +import { observeTrackReference$ } from "../state/MediaViewModel"; +import { useUrlParams } from "../UrlParams"; +import { useInitial } from "../useInitial"; +import { getValue } from "../utils/observable"; +import { type SelectedDevice } from "../state/MediaDevices"; + +interface UseLivekitResult { + livekitPublicationRoom?: Room; + connState: ECConnectionState; +} + +// TODO-MULTI-SFU This is all the logic we need in the subscription connection logic (sync output devices) +// This is not used! (but summarizes what we need) +export function livekitSubscriptionRoom( + rtcSession: MatrixRTCSession, + muteStates: MuteStates, + sfuConfig: SFUConfig | undefined, + e2eeSystem: EncryptionSystem, +): UseLivekitResult { + // Only ever create the room once via useInitial. + // The call can end up with multiple livekit rooms. This is the particular room in + // which this participant publishes their media. + const publicationRoom = useInitial(() => { + logger.info("[LivekitRoom] Create LiveKit room"); + + let e2ee: E2EEManagerOptions | undefined; + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + logger.info("Created MatrixKeyProvider (per participant)"); + e2ee = { + keyProvider: new MatrixKeyProvider(), + worker: new E2EEWorker(), + }; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + logger.info("Created ExternalE2EEKeyProvider (shared key)"); + e2ee = { + keyProvider: new ExternalE2EEKeyProvider(), + worker: new E2EEWorker(), + }; + } + + const roomOptions: RoomOptions = { + ...defaultLiveKitOptions, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id, + }, + e2ee, + }; + // We have to create the room manually here due to a bug inside + // @livekit/components-react. JSON.stringify() is used in deps of a + // useEffect() with an argument that references itself, if E2EE is enabled + const room = new Room(roomOptions); + room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + + return room; + }); + + // Setup and update the keyProvider which was create by `createRoom` + useEffect(() => { + const e2eeOptions = publicationRoom.options.e2ee; + if ( + e2eeSystem.kind === E2eeType.NONE || + !(e2eeOptions && "keyProvider" in e2eeOptions) + ) + return; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession); + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + (e2eeOptions.keyProvider as ExternalE2EEKeyProvider) + .setKey(e2eeSystem.secret) + .catch((e) => { + logger.error("Failed to set shared key for E2EE", e); + }); + } + }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); + + return { + connState: connectionState, + livekitPublicationRoom: publicationRoom, + }; +} diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 2ebd6045..a288ec57 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -7,12 +7,7 @@ Please see LICENSE in the repository root for full details. import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { useEffect, useState } from "react"; -import { type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; -import { useActiveLivekitFocus } from "../room/useActiveFocus"; -import { useErrorBoundary } from "../useErrorBoundary"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; @@ -34,38 +29,11 @@ export type OpenIDClientParts = Pick< "getOpenIdToken" | "getDeviceId" >; -export function useOpenIDSFU( - client: OpenIDClientParts, - rtcSession: MatrixRTCSession, -): SFUConfig | undefined { - const [sfuConfig, setSFUConfig] = useState(undefined); - - const activeFocus = useActiveLivekitFocus(rtcSession); - const { showErrorBoundary } = useErrorBoundary(); - - useEffect(() => { - if (activeFocus) { - getSFUConfigWithOpenID(client, activeFocus).then( - (sfuConfig) => { - setSFUConfig(sfuConfig); - }, - (e) => { - showErrorBoundary(new FailToGetOpenIdToken(e)); - logger.error("Failed to get SFU config", e); - }, - ); - } else { - setSFUConfig(undefined); - } - }, [client, activeFocus, showErrorBoundary]); - - return sfuConfig; -} - export async function getSFUConfigWithOpenID( client: OpenIDClientParts, - activeFocus: LivekitFocus, -): Promise { + serviceUrl: string, + livekitAlias: string, +): Promise { let openIdToken: IOpenIDToken; try { openIdToken = await doNetworkOperationWithRetry(async () => @@ -78,26 +46,16 @@ export async function getSFUConfigWithOpenID( } logger.debug("Got openID token", openIdToken); - try { - logger.info( - `Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...`, - ); - const sfuConfig = await getLiveKitJWT( - client, - activeFocus.livekit_service_url, - activeFocus.livekit_alias, - openIdToken, - ); - logger.info(`Got JWT from call's active focus URL.`); + logger.info(`Trying to get JWT for focus ${serviceUrl}...`); + const sfuConfig = await getLiveKitJWT( + client, + serviceUrl, + livekitAlias, + openIdToken, + ); + logger.info(`Got JWT from call's active focus URL.`); - return sfuConfig; - } catch (e) { - logger.warn( - `Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`, - e, - ); - return undefined; - } + return sfuConfig; } async function getLiveKitJWT( diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts index 4c669b47..0672a8eb 100644 --- a/src/livekit/useLivekit.ts +++ b/src/livekit/useLivekit.ts @@ -50,11 +50,12 @@ import { getValue } from "../utils/observable"; import { type SelectedDevice } from "../state/MediaDevices"; interface UseLivekitResult { - livekitRoom?: Room; + livekitPublicationRoom?: Room; connState: ECConnectionState; } -export function useLivekit( +// TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object. +export function useLivekitPublicationRoom( rtcSession: MatrixRTCSession, muteStates: MuteStates, sfuConfig: SFUConfig | undefined, @@ -83,7 +84,9 @@ export function useLivekit( const { processor } = useTrackProcessor(); // Only ever create the room once via useInitial. - const room = useInitial(() => { + // The call can end up with multiple livekit rooms. This is the particular room in + // which this participant publishes their media. + const publicationRoom = useInitial(() => { logger.info("[LivekitRoom] Create LiveKit room"); let e2ee: E2EEManagerOptions | undefined; @@ -135,7 +138,7 @@ export function useLivekit( // Setup and update the keyProvider which was create by `createRoom` useEffect(() => { - const e2eeOptions = room.options.e2ee; + const e2eeOptions = publicationRoom.options.e2ee; if ( e2eeSystem.kind === E2eeType.NONE || !(e2eeOptions && "keyProvider" in e2eeOptions) @@ -151,7 +154,7 @@ export function useLivekit( logger.error("Failed to set shared key for E2EE", e); }); } - }, [room.options.e2ee, e2eeSystem, rtcSession]); + }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); // Sync the requested track processors with LiveKit useTrackProcessorSync( @@ -170,7 +173,7 @@ export function useLivekit( return track instanceof LocalVideoTrack ? track : null; }), ), - [room], + [publicationRoom], ), ), ); @@ -178,7 +181,7 @@ export function useLivekit( const connectionState = useECConnectionState( initialAudioInputId, initialMuteStates.audio.enabled, - room, + publicationRoom, sfuConfig, ); @@ -216,8 +219,11 @@ export function useLivekit( // It's important that we only do this in the connected state, because // LiveKit's internal mute states aren't consistent during connection setup, // and setting tracks to be enabled during this time causes errors. - if (room !== undefined && connectionState === ConnectionState.Connected) { - const participant = room.localParticipant; + if ( + publicationRoom !== undefined && + connectionState === ConnectionState.Connected + ) { + const participant = publicationRoom.localParticipant; // Always update the muteButtonState Ref so that we can read the current // state in awaited blocks. buttonEnabled.current = { @@ -275,7 +281,7 @@ export function useLivekit( audioMuteUpdating.current = true; trackPublication = await participant.setMicrophoneEnabled( buttonEnabled.current.audio, - room.options.audioCaptureDefaults, + publicationRoom.options.audioCaptureDefaults, ); audioMuteUpdating.current = false; break; @@ -283,7 +289,7 @@ export function useLivekit( videoMuteUpdating.current = true; trackPublication = await participant.setCameraEnabled( buttonEnabled.current.video, - room.options.videoCaptureDefaults, + publicationRoom.options.videoCaptureDefaults, ); videoMuteUpdating.current = false; break; @@ -347,11 +353,14 @@ export function useLivekit( logger.error("Failed to sync video mute state with LiveKit", e); }); } - }, [room, muteStates, connectionState]); + }, [publicationRoom, muteStates, connectionState]); useEffect(() => { // Sync the requested devices with LiveKit's devices - if (room !== undefined && connectionState === ConnectionState.Connected) { + if ( + publicationRoom !== undefined && + connectionState === ConnectionState.Connected + ) { const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, @@ -359,15 +368,15 @@ export function useLivekit( selected$.subscribe((device) => { logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - room.getActiveDevice(kind), + publicationRoom.getActiveDevice(kind), " !== ", device?.id, ); if ( device !== undefined && - room.getActiveDevice(kind) !== device.id + publicationRoom.getActiveDevice(kind) !== device.id ) { - room + publicationRoom .switchActiveDevice(kind, device.id) .catch((e) => logger.error(`Failed to sync ${kind} device with LiveKit`, e), @@ -393,7 +402,7 @@ export function useLivekit( .pipe(switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER)) .subscribe(() => { const activeMicTrack = Array.from( - room.localParticipant.audioTrackPublications.values(), + publicationRoom.localParticipant.audioTrackPublications.values(), ).find((d) => d.source === Track.Source.Microphone)?.track; if ( @@ -408,7 +417,7 @@ export function useLivekit( // getUserMedia() call with deviceId: default to get the *new* default device. // Note that room.switchActiveDevice() won't work: Livekit will ignore it because // the deviceId hasn't changed (was & still is default). - room.localParticipant + publicationRoom.localParticipant .getTrackPublication(Track.Source.Microphone) ?.audioTrack?.restartTrack() .catch((e) => { @@ -422,10 +431,10 @@ export function useLivekit( for (const s of subscriptions) s?.unsubscribe(); }; } - }, [room, devices, connectionState, controlledAudioDevices]); + }, [publicationRoom, devices, connectionState, controlledAudioDevices]); return { connState: connectionState, - livekitRoom: room, + livekitPublicationRoom: publicationRoom, }; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 6cdbb75c..e12fc060 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -5,9 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { IconButton, Text, Tooltip } from "@vector-im/compound-web"; -import { ConnectionState, type Room as LivekitRoom } from "livekit-client"; import { type MatrixClient, type Room as MatrixRoom } from "matrix-js-sdk"; import { type FC, @@ -37,6 +35,7 @@ import { VolumeOnSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; +import { ConnectionState } from "livekit-client"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -59,14 +58,12 @@ import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; -import { useLivekit } from "../livekit/useLivekit.ts"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { type MuteStates } from "./MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; -import { useOpenIDSFU } from "../livekit/openIDSFU"; import { CallViewModel, type GridMode, @@ -108,9 +105,7 @@ import { useSetting, } from "../settings/settings"; import { ReactionsReader } from "../reactions/ReactionsReader"; -import { ConnectionLostError } from "../utils/errors.ts"; import { useTypedEventEmitter } from "../useEvents.ts"; -import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships.ts"; import { useMediaDevices } from "../MediaDevicesContext.ts"; @@ -125,7 +120,7 @@ import { prefetchSounds } from "../soundUtils"; import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; -import { ObservableScope } from "../state/ObservableScope.ts"; +import { ConnectionLostError } from "../utils/errors.ts"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -138,92 +133,47 @@ export interface ActiveCallProps export const ActiveCall: FC = (props) => { const mediaDevices = useMediaDevices(); - const sfuConfig = useOpenIDSFU(props.client, props.rtcSession); - const { livekitRoom, connState } = useLivekit( - props.rtcSession, - props.muteStates, - sfuConfig, - props.e2eeSystem, - ); - const observableScope = useInitial(() => new ObservableScope()); - const connStateBehavior$ = useObservable( - (inputs$) => - observableScope.behavior( - inputs$.pipe(map(([connState]) => connState)), - connState, - ), - [connState], - ); const [vm, setVm] = useState(null); - useEffect(() => { - logger.info( - `[Lifecycle] InCallView Component mounted, livekit room state ${livekitRoom?.state}`, - ); - return (): void => { - logger.info( - `[Lifecycle] InCallView Component unmounted, livekit room state ${livekitRoom?.state}`, - ); - livekitRoom - ?.disconnect() - .then(() => { - logger.info( - `[Lifecycle] Disconnected from livekit room, state:${livekitRoom?.state}`, - ); - }) - .catch((e) => { - logger.error("[Lifecycle] Failed to disconnect from livekit room", e); - }); - }; - }, [livekitRoom]); - - const { autoLeaveWhenOthersLeft, sendNotificationType, waitForCallPickup } = + const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = useUrlParams(); useEffect(() => { - if (livekitRoom !== undefined) { - const reactionsReader = new ReactionsReader(props.rtcSession); - const vm = new CallViewModel( - props.rtcSession, - props.matrixRoom, - livekitRoom, - mediaDevices, - { - encryptionSystem: props.e2eeSystem, - autoLeaveWhenOthersLeft, - waitForCallPickup: - waitForCallPickup && sendNotificationType === "ring", - }, - connStateBehavior$, - reactionsReader.raisedHands$, - reactionsReader.reactions$, - ); - setVm(vm); - return (): void => { - vm.destroy(); - reactionsReader.destroy(); - }; - } + const reactionsReader = new ReactionsReader(props.rtcSession); + const vm = new CallViewModel( + props.rtcSession, + props.matrixRoom, + mediaDevices, + { + encryptionSystem: props.e2eeSystem, + autoLeaveWhenOthersLeft, + waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", + }, + reactionsReader.raisedHands$, + reactionsReader.reactions$, + props.e2eeSystem, + ); + setVm(vm); + return (): void => { + vm.destroy(); + reactionsReader.destroy(); + }; }, [ props.rtcSession, props.matrixRoom, - livekitRoom, mediaDevices, props.e2eeSystem, - connStateBehavior$, autoLeaveWhenOthersLeft, sendNotificationType, waitForCallPickup, ]); - if (livekitRoom === undefined || vm === null) return null; + if (vm === null) return null; return ( - - - - - + + + ); }; @@ -233,7 +183,6 @@ export interface InCallViewProps { matrixInfo: MatrixInfo; rtcSession: MatrixRTCSession; matrixRoom: MatrixRoom; - livekitRoom: LivekitRoom; muteStates: MuteStates; /** Function to call when the user explicitly ends the call */ onLeave: (cause: "user", soundFile?: CallEventSounds) => void; @@ -248,7 +197,6 @@ export const InCallView: FC = ({ matrixInfo, rtcSession, matrixRoom, - livekitRoom, muteStates, onLeave, header: headerStyle, @@ -273,10 +221,6 @@ export const InCallView: FC = ({ const { hideScreensharing, showControls } = useUrlParams(); - const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ - room: livekitRoom, - }); - const muteAllAudio = useBehavior(muteAllAudio$); // Call pickup state and display names are needed for waiting overlay/sounds const callPickupState = useBehavior(vm.callPickupState$); @@ -806,15 +750,16 @@ export const InCallView: FC = ({ ); const toggleScreensharing = useCallback(() => { - localParticipant - .setScreenShareEnabled(!isScreenShareEnabled, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error); - }, [localParticipant, isScreenShareEnabled]); + throw new Error("TODO-MULTI-SFU"); + // localParticipant + // .setScreenShareEnabled(!isScreenShareEnabled, { + // audio: true, + // selfBrowserSurface: "include", + // surfaceSwitching: "include", + // systemAudio: "include", + // }) + // .catch(logger.error); + }, []); const buttons: JSX.Element[] = []; @@ -841,7 +786,7 @@ export const InCallView: FC = ({ = ({ ) } - + {/* TODO-MULTI-SFU: */} {renderContent()} @@ -955,7 +900,7 @@ export const InCallView: FC = ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - livekitRoom={livekitRoom} + livekitRoom={undefined} // TODO-MULTI-SFU /> )} diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 73f58cea..e5e567ef 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { isLivekitFocus, isLivekitFocusConfig, + LivekitFocusConfig, type LivekitFocus, type LivekitFocusActive, type MatrixRTCSession, @@ -31,24 +32,16 @@ export function makeActiveFocus(): LivekitFocusActive { }; } -async function makePreferredLivekitFoci( +export function getLivekitAlias(rtcSession: MatrixRTCSession): string { + // For now we assume everything is a room-scoped call + return rtcSession.room.roomId; +} + +async function makeFocusInternal( rtcSession: MatrixRTCSession, - livekitAlias: string, -): Promise { - logger.log("Start building foci_preferred list: ", rtcSession.room.roomId); - - const preferredFoci: LivekitFocus[] = []; - - // Make the Focus from the running rtc session the highest priority one - // This minimizes how often we need to switch foci during a call. - const focusInUse = rtcSession.getFocusInUse(); - if (focusInUse && isLivekitFocus(focusInUse)) { - logger.log("Adding livekit focus from oldest member: ", focusInUse); - preferredFoci.push(focusInUse); - } - - // Warm up the first focus we owned, to ensure livekit room is created before any state event sent. - let toWarmUp: LivekitFocus | undefined; +): Promise { + logger.log("Searching for a preferred focus"); + const livekitAlias = getLivekitAlias(rtcSession); // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); @@ -59,51 +52,42 @@ async function makePreferredLivekitFoci( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - const validWellKnownFoci = wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - logger.log("Adding livekit focus from well known: ", wellKnownFocus); - return { ...wellKnownFocus, livekit_alias: livekitAlias }; - }); - if (validWellKnownFoci.length > 0) { - toWarmUp = validWellKnownFoci[0]; + const focus: LivekitFocusConfig | undefined = wellKnownFoci.find( + (f) => f && isLivekitFocusConfig(f), + ); + if (focus !== undefined) { + logger.log("Using LiveKit focus from .well-known: ", focus); + return { ...focus, livekit_alias: livekitAlias }; } - preferredFoci.push(...validWellKnownFoci); } } const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { - const focusFormConf: LivekitFocus = { + const focusFromConf: LivekitFocus = { type: "livekit", livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - toWarmUp = toWarmUp ?? focusFormConf; - logger.log("Adding livekit focus from config: ", focusFormConf); - preferredFoci.push(focusFormConf); + logger.log("Using LiveKit focus from config: ", focusFromConf); + return focusFromConf; } - if (toWarmUp) { - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, toWarmUp); - } - if (preferredFoci.length === 0) - throw new MatrixRTCFocusMissingError(domain ?? ""); - return Promise.resolve(preferredFoci); + throw new MatrixRTCFocusMissingError(domain ?? ""); +} - // TODO: we want to do something like this: - // - // const focusOtherMembers = await focusFromOtherMembers( - // rtcSession, - // livekitAlias, - // ); - // if (focusOtherMembers) preferredFoci.push(focusOtherMembers); +export async function makeFocus( + rtcSession: MatrixRTCSession, +): Promise { + const focus = await makeFocusInternal(rtcSession); + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + await getSFUConfigWithOpenID(rtcSession.room.client, focus); + return focus; } export async function enterRTCSession( rtcSession: MatrixRTCSession, + focus: LivekitFocus, encryptMedia: boolean, useNewMembershipManager = true, useExperimentalToDeviceTransport = false, @@ -115,34 +99,27 @@ export async function enterRTCSession( // have started tracking by the time calls start getting created. // groupCallOTelMembership?.onJoinCall(); - // right now we assume everything is a room-scoped call - const livekitAlias = rtcSession.room.roomId; const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; - rtcSession.joinRoomSession( - await makePreferredLivekitFoci(rtcSession, livekitAlias), - makeActiveFocus(), - { - notificationType: getUrlParams().sendNotificationType, - useNewMembershipManager, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: - matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport, - }, - ); + rtcSession.joinRoomSession([focus], focus, { + notificationType: getUrlParams().sendNotificationType, + useNewMembershipManager, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport, + }); if (widget) { try { await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 462e4afc..f0d2c0b7 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -12,7 +12,9 @@ import { } from "@livekit/components-core"; import { ConnectionState, - type Room as LivekitRoom, + E2EEOptions, + ExternalE2EEKeyProvider, + Room as LivekitRoom, type LocalParticipant, ParticipantEvent, type RemoteParticipant, @@ -22,6 +24,7 @@ import { type EventTimelineSetHandlerMap, EventType, RoomEvent, + MatrixClient, RoomStateEvent, SyncState, type Room as MatrixRoom, @@ -63,6 +66,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, + isLivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -116,7 +120,16 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { constant, type Behavior } from "./Behavior"; +import { type Behavior } from "./Behavior"; +import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; +import { defaultLiveKitOptions } from "../livekit/options"; +import { + enterRTCSession, + getLivekitAlias, + makeFocus, +} from "../rtcSessionHelpers"; +import { E2eeType } from "../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -405,6 +418,31 @@ class ScreenShare { type MediaItem = UserMedia | ScreenShare; +function getE2eeOptions( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): E2EEOptions | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } +} + function getRoomMemberFromRtcMember( rtcMember: CallMembership, room: MatrixRoom, @@ -427,8 +465,151 @@ function getRoomMemberFromRtcMember( return { id, member }; } -// TODO: Move wayyyy more business logic from the call and lobby views into here +class Connection { + // TODO-MULTI-SFU Add all device syncing logic from useLivekit + private readonly sfuConfig = getSFUConfigWithOpenID( + this.client, + this.serviceUrl, + this.livekitAlias, + ); + + public async startSubscribing(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + } + + public async startPublishing(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) + // TODO-MULTI-SFU this should not create a track? + await this.livekitRoom.localParticipant.createTracks({ + audio: { deviceId: "default" }, + }); + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + } + + private stopped = false; + + public stop(): void { + void this.livekitRoom.disconnect(); + this.stopped = true; + } + + public readonly participants$ = connectedParticipantsObserver( + this.livekitRoom, + ).pipe(this.scope.state()); + + public constructor( + private readonly livekitRoom: LivekitRoom, + private readonly serviceUrl: string, + private readonly livekitAlias: string, + private readonly client: MatrixClient, + private readonly scope: ObservableScope, + ) {} +} + export class CallViewModel extends ViewModel { + private readonly e2eeOptions = getE2eeOptions( + this.encryptionSystem, + this.matrixRTCSession, + ); + + private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); + + private readonly livekitRoom = new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: this.e2eeOptions, + }); + + private readonly localFocus = makeFocus(this.matrixRTCSession); + + private readonly localConnection = this.localFocus.then( + (focus) => + new Connection( + this.livekitRoom, + focus.livekit_service_url, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + ), + ); + + private readonly memberships$ = fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(map(() => this.matrixRTCSession.memberships)); + + private readonly foci$ = this.memberships$.pipe( + map( + (memberships) => + new Set( + memberships + .map((m) => this.matrixRTCSession.resolveActiveFocus(m)) + .filter((f) => f !== undefined && isLivekitFocusConfig(f)) + .map((f) => f.livekit_service_url), + ), + ), + ); + + private readonly remoteConnections$ = combineLatest([ + this.localFocus, + this.foci$, + ]).pipe( + accumulate(new Map(), (prev, [localFocus, foci]) => { + const stopped = new Map(prev); + const next = new Map(); + + for (const focus of foci) { + if (focus !== localFocus.livekit_service_url) { + stopped.delete(focus); + next.set( + focus, + prev.get(focus) ?? + new Connection( + new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: this.e2eeOptions, + }), + focus, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + ), + ); + } + } + + for (const connection of stopped.values()) connection.stop(); + return next; + }), + ); + + private readonly joined$ = new Subject(); + + public join(): void { + this.joined$.next(); + } + + public leave(): void { + // TODO + } + + private readonly connectionInstructions$ = this.joined$.pipe( + switchMap(() => this.remoteConnections$), + startWith(new Map()), + pairwise(), + map(([prev, next]) => { + const start = new Set(next.values()); + for (const connection of prev.values()) start.delete(connection); + const stop = new Set(prev.values()); + for (const connection of next.values()) stop.delete(connection); + + return { start, stop }; + }), + ); + private readonly userId = this.matrixRoom.client.getUserId(); private readonly matrixConnected$ = this.scope.behavior( @@ -502,79 +683,13 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - /** - * The raw list of RemoteParticipants as reported by LiveKit - */ - private readonly rawRemoteParticipants$ = this.scope.behavior< - RemoteParticipant[] - >(connectedParticipantsObserver(this.livekitRoom), []); - - /** - * Lists of RemoteParticipants to "hold" on display, even if LiveKit claims that - * they've left - */ - private readonly remoteParticipantHolds$ = this.scope.behavior< - RemoteParticipant[][] - >( - this.livekitConnectionState$.pipe( - withLatestFrom(this.rawRemoteParticipants$), - mergeMap(([s, ps]) => { - // Whenever we switch focuses, we should retain all the previous - // participants for at least POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS ms to - // give their clients time to switch over and avoid jarring layout shifts - if (s === ECAddonConnectionState.ECSwitchingFocus) { - return concat( - // Hold these participants - of({ hold: ps }), - // Wait for time to pass and the connection state to have changed - forkJoin([ - timer(POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS), - this.livekitConnectionState$.pipe( - filter((s) => s !== ECAddonConnectionState.ECSwitchingFocus), - take(1), - ), - // Then unhold them - ]).pipe(map(() => ({ unhold: ps }))), - ); - } else { - return EMPTY; - } - }), - // Accumulate the hold instructions into a single list showing which - // participants are being held - accumulate([] as RemoteParticipant[][], (holds, instruction) => - "hold" in instruction - ? [instruction.hold, ...holds] - : holds.filter((h) => h !== instruction.unhold), - ), - ), - ); - /** * The RemoteParticipants including those that are being "held" on the screen */ private readonly remoteParticipants$ = this.scope - .behavior( - combineLatest( - [this.rawRemoteParticipants$, this.remoteParticipantHolds$], - (raw, holds) => { - const result = [...raw]; - const resultIds = new Set(result.map((p) => p.identity)); - - // Incorporate the held participants into the list - for (const hold of holds) { - for (const p of hold) { - if (!resultIds.has(p.identity)) { - result.push(p); - resultIds.add(p.identity); - } - } - } - - return result; - }, - ), - ) + .behavior< + RemoteParticipant[] + >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participants$, ...[...remoteConnections.values()].map((c) => c.participants$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) .pipe(pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( @@ -1685,24 +1800,42 @@ export class CallViewModel extends ViewModel { ), filter((v) => v.playSounds), ); + // TODO-REBASE: expose connection state observable + public readonly livekitConnectionState$: Observable; public constructor( - // A call is permanently tied to a single Matrix room and LiveKit room + // A call is permanently tied to a single Matrix room private readonly matrixRTCSession: MatrixRTCSession, private readonly matrixRoom: MatrixRoom, - private readonly livekitRoom: LivekitRoom, private readonly mediaDevices: MediaDevices, private readonly options: CallViewModelOptions, - public readonly livekitConnectionState$: Behavior, private readonly handsRaisedSubject$: Observable< Record >, private readonly reactionsSubject$: Observable< Record >, + private readonly encryptionSystem: EncryptionSystem, ) { super(); + void this.localConnection.then((c) => c.startPublishing()); + this.connectionInstructions$ + .pipe(this.scope.bind()) + .subscribe(({ start, stop }) => { + for (const connection of start) connection.startSubscribing(); + for (const connection of stop) connection.stop(); + }); + combineLatest([this.localFocus, this.joined$]) + .pipe(this.scope.bind()) + .subscribe(([localFocus]) => { + enterRTCSession( + this.matrixRTCSession, + localFocus, + this.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, + ); + }); + // Pause upstream of all local media tracks when we're disconnected from // MatrixRTC, because it can be an unpleasant surprise for the app to say // 'reconnecting' and yet still be transmitting your media to others. diff --git a/yarn.lock b/yarn.lock index 5f224576..668706b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10317,7 +10317,7 @@ __metadata: uuid: "npm:11" checksum: 10c0/ecd019c677c272c5598617dcde407dbe4b1b11460863b2a577e33f3fd8732c9d9073ec0221b471ec1eb24e2839eec20728db7f92c9348be83126547286e50805 languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": version: 1.13.1 From 35319dd6b507bc6e0ec9552e285f31a95dc56d94 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 14:29:22 +0200 Subject: [PATCH 002/144] Fix some errors in CallViewModel --- src/state/CallViewModel.ts | 39 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index f0d2c0b7..11ca2428 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -12,19 +12,20 @@ import { } from "@livekit/components-core"; import { ConnectionState, - E2EEOptions, + type E2EEOptions, ExternalE2EEKeyProvider, Room as LivekitRoom, type LocalParticipant, ParticipantEvent, type RemoteParticipant, } from "livekit-client"; +import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { ClientEvent, type EventTimelineSetHandlerMap, EventType, RoomEvent, - MatrixClient, + type MatrixClient, RoomStateEvent, SyncState, type Room as MatrixRoom, @@ -41,12 +42,10 @@ import { distinctUntilChanged, endWith, filter, - forkJoin, fromEvent, ignoreElements, map, merge, - mergeMap, of, pairwise, race, @@ -61,7 +60,6 @@ import { takeUntil, throttleTime, timer, - withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -75,10 +73,6 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { ViewModel } from "./ViewModel"; -import { - ECAddonConnectionState, - type ECConnectionState, -} from "../livekit/useECConnectionState"; import { LocalUserMediaViewModel, type MediaViewModel, @@ -120,7 +114,7 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { type Behavior } from "./Behavior"; +import { constant, type Behavior } from "./Behavior"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { defaultLiveKitOptions } from "../livekit/options"; import { @@ -130,6 +124,7 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; +import { ECConnectionState } from "../livekit/useECConnectionState"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -141,10 +136,6 @@ export interface CallViewModelOptions { waitForCallPickup?: boolean; } -// How long we wait after a focus switch before showing the real participant -// list again -const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; - // Do not play any sounds if the participant count has exceeded this // number. export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; @@ -497,9 +488,10 @@ class Connection { this.stopped = true; } - public readonly participants$ = connectedParticipantsObserver( - this.livekitRoom, - ).pipe(this.scope.state()); + public readonly participants$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); public constructor( private readonly livekitRoom: LivekitRoom, @@ -649,9 +641,10 @@ export class CallViewModel extends ViewModel { private readonly connected$ = this.scope.behavior( and$( this.matrixConnected$, - this.livekitConnectionState$.pipe( - map((state) => state === ConnectionState.Connected), - ), + // TODO-MULTI-SFU + // this.livekitConnectionState$.pipe( + // map((state) => state === ConnectionState.Connected), + // ), ), ); @@ -1819,17 +1812,17 @@ export class CallViewModel extends ViewModel { ) { super(); - void this.localConnection.then((c) => c.startPublishing()); + void this.localConnection.then((c) => void c.startPublishing()); this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { - for (const connection of start) connection.startSubscribing(); + for (const connection of start) void connection.startSubscribing(); for (const connection of stop) connection.stop(); }); combineLatest([this.localFocus, this.joined$]) .pipe(this.scope.bind()) .subscribe(([localFocus]) => { - enterRTCSession( + void enterRTCSession( this.matrixRTCSession, localFocus, this.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, From d9b6302bf769b51c34d92d5f44759226dd2df085 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 14:36:13 +0200 Subject: [PATCH 003/144] Fix crash? --- src/state/CallViewModel.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 11ca2428..468bbd3f 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -504,7 +504,7 @@ class Connection { export class CallViewModel extends ViewModel { private readonly e2eeOptions = getE2eeOptions( - this.encryptionSystem, + this.options.encryptionSystem, this.matrixRTCSession, ); @@ -1808,7 +1808,6 @@ export class CallViewModel extends ViewModel { private readonly reactionsSubject$: Observable< Record >, - private readonly encryptionSystem: EncryptionSystem, ) { super(); @@ -1825,7 +1824,7 @@ export class CallViewModel extends ViewModel { void enterRTCSession( this.matrixRTCSession, localFocus, - this.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, + this.options.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, ); }); From 376a4b4e4a538a226deea636accbe84176f994cb Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 15:06:14 +0200 Subject: [PATCH 004/144] initial compiling version Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 47 ++++++++++++++++++-------------------- src/room/RoomPage.tsx | 9 ++++---- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index dbc3ea18..8562d4f8 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -41,7 +41,7 @@ import { ActiveCall } from "./InCallView"; import { MUTE_PARTICIPANT_COUNT, type MuteStates } from "./MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; -import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers"; +import { leaveRTCSession } from "../rtcSessionHelpers"; import { saveKeyForRoom, useRoomEncryptionSystem, @@ -90,7 +90,8 @@ interface Props { skipLobby: UrlParams["skipLobby"]; header: HeaderStyle; rtcSession: MatrixRTCSession; - isJoined: boolean; + joined: boolean; + setJoined: (value: boolean) => void; muteStates: MuteStates; widget: WidgetHelpers | null; } @@ -103,7 +104,8 @@ export const GroupCallView: FC = ({ skipLobby, header, rtcSession, - isJoined, + joined, + setJoined, muteStates, widget, }) => { @@ -213,12 +215,14 @@ export const GroupCallView: FC = ({ const enterRTCSessionOrError = useCallback( async (rtcSession: MatrixRTCSession): Promise => { try { - await enterRTCSession( - rtcSession, - perParticipantE2EE, - useNewMembershipManager, - useExperimentalToDeviceTransport, - ); + setJoined(true); + // TODO-MULTI-SFU what to do with error handling now that we don't use this function? + // await enterRTCSession( + // rtcSession, + // perParticipantE2EE, + // useNewMembershipManager, + // useExperimentalToDeviceTransport, + // ); } catch (e) { if (e instanceof ElementCallError) { setExternalError(e); @@ -230,12 +234,9 @@ export const GroupCallView: FC = ({ setExternalError(error); } } + return Promise.resolve(); }, - [ - perParticipantE2EE, - useExperimentalToDeviceTransport, - useNewMembershipManager, - ], + [setJoined], ); useEffect(() => { @@ -284,7 +285,7 @@ export const GroupCallView: FC = ({ const onJoin = (ev: CustomEvent): void => { (async (): Promise => { await defaultDeviceSetup(ev.detail.data as unknown as JoinCallData); - await enterRTCSessionOrError(rtcSession); + setJoined(true); widget.api.transport.reply(ev.detail, {}); })().catch((e) => { logger.error("Error joining RTC session on preload", e); @@ -296,11 +297,7 @@ export const GroupCallView: FC = ({ }; } else { // No lobby and no preload: we enter the rtc session right away - (async (): Promise => { - await enterRTCSessionOrError(rtcSession); - })().catch((e) => { - logger.error("Error joining RTC session immediately", e); - }); + setJoined(true); } } }, [ @@ -311,7 +308,7 @@ export const GroupCallView: FC = ({ perParticipantE2EE, mediaDevices, latestMuteStates, - enterRTCSessionOrError, + setJoined, useNewMembershipManager, ]); @@ -373,7 +370,7 @@ export const GroupCallView: FC = ({ ); useEffect(() => { - if (widget && isJoined) { + if (widget && joined) { // set widget to sticky once joined. widget.api.setAlwaysOnScreen(true).catch((e) => { logger.error("Error calling setAlwaysOnScreen(true)", e); @@ -391,7 +388,7 @@ export const GroupCallView: FC = ({ widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); }; } - }, [widget, isJoined, rtcSession]); + }, [widget, joined, rtcSession]); const joinRule = useJoinRule(room); @@ -426,7 +423,7 @@ export const GroupCallView: FC = ({ client={client} matrixInfo={matrixInfo} muteStates={muteStates} - onEnter={() => void enterRTCSessionOrError(rtcSession)} + onEnter={() => setJoined(true)} confineToRoom={confineToRoom} hideHeader={header === HeaderStyle.None} participantCount={participantCount} @@ -444,7 +441,7 @@ export const GroupCallView: FC = ({ throw externalError; }; body = ; - } else if (isJoined) { + } else if (joined) { body = ( <> {shareModal} diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index b424c511..480f8706 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -61,10 +61,8 @@ export const RoomPage: FC = () => { const { avatarUrl, displayName: userDisplayName } = useProfile(client); const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers); - const isJoined = useMatrixRTCSessionJoinState( - groupCallState.kind === "loaded" ? groupCallState.rtcSession : undefined, - ); - const muteStates = useMuteStates(isJoined); + const [joined, setJoined] = useState(false); + const muteStates = useMuteStates(joined); useEffect(() => { // If we've finished loading, are not already authed and we've been given a display name as @@ -109,7 +107,8 @@ export const RoomPage: FC = () => { widget={widget} client={client!} rtcSession={groupCallState.rtcSession} - isJoined={isJoined} + joined={joined} + setJoined={setJoined} isPasswordlessUser={passwordlessUser} confineToRoom={confineToRoom} preload={preload} From c6e8c94fd6c59c7d29025302aef4618a34c5f6dd Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 15:07:55 +0200 Subject: [PATCH 005/144] Fix makeFocus --- src/rtcSessionHelpers.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index e5e567ef..c08fcd40 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -6,11 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { - isLivekitFocus, isLivekitFocusConfig, - LivekitFocusConfig, + type LivekitFocusConfig, type LivekitFocus, - type LivekitFocusActive, + type LivekitFocusSelection, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -25,7 +24,7 @@ import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; -export function makeActiveFocus(): LivekitFocusActive { +export function makeActiveFocus(): LivekitFocusSelection { return { type: "livekit", focus_selection: "oldest_membership", @@ -81,7 +80,11 @@ export async function makeFocus( ): Promise { const focus = await makeFocusInternal(rtcSession); // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, focus); + await getSFUConfigWithOpenID( + rtcSession.room.client, + focus.livekit_service_url, + focus.livekit_alias, + ); return focus; } From cb91f1ad4ffe043d321f5ce9ef3a4895ce81483b Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 15:33:41 +0200 Subject: [PATCH 006/144] Make it actually join the session --- src/state/CallViewModel.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 468bbd3f..623e9e9e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -531,7 +531,10 @@ export class CallViewModel extends ViewModel { private readonly memberships$ = fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, - ).pipe(map(() => this.matrixRTCSession.memberships)); + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.memberships), + ); private readonly foci$ = this.memberships$.pipe( map( @@ -1873,5 +1876,8 @@ export class CallViewModel extends ViewModel { } } }); + + // Join automatically + this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } } From 7b88420f6a255e4956e84481edaf609d178a0048 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 16:56:57 +0200 Subject: [PATCH 007/144] first video! Signed-off-by: Timo K --- src/main.tsx | 6 +++--- src/room/InCallView.tsx | 2 -- src/state/CallViewModel.ts | 29 +++++++++++++++++++++++------ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index 06275f59..e795a13c 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -60,9 +60,9 @@ if (fatalError !== null) { Initializer.initBeforeReact() .then(() => { root.render( - - - , + // + , + // , ); }) .catch((e) => { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e12fc060..1c8b41e9 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -107,7 +107,6 @@ import { import { ReactionsReader } from "../reactions/ReactionsReader"; import { useTypedEventEmitter } from "../useEvents.ts"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; -import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships.ts"; import { useMediaDevices } from "../MediaDevicesContext.ts"; import { EarpieceOverlay } from "./EarpieceOverlay.tsx"; import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx"; @@ -251,7 +250,6 @@ export const InCallView: FC = ({ useExperimentalToDeviceTransportSetting, ); const encryptionSystem = useRoomEncryptionSystem(matrixRoom.roomId); - const memberships = useMatrixRTCSessionMemberships(rtcSession); const showToDeviceEncryption = useMemo( () => diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 623e9e9e..bade6ef9 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -42,6 +42,7 @@ import { distinctUntilChanged, endWith, filter, + from, fromEvent, ignoreElements, map, @@ -367,6 +368,7 @@ class UserMedia { public destroy(): void { this.scope.end(); + this.vm.destroy(); } } @@ -473,12 +475,17 @@ class Connection { public async startPublishing(): Promise { this.stopped = false; const { url, jwt } = await this.sfuConfig; - if (!this.stopped) - // TODO-MULTI-SFU this should not create a track? - await this.livekitRoom.localParticipant.createTracks({ - audio: { deviceId: "default" }, - }); if (!this.stopped) await this.livekitRoom.connect(url, jwt); + if (!this.stopped) { + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: { deviceId: "default" }, + video: true, + }); + for (const track of tracks) { + await this.livekitRoom.localParticipant.publishTrack(track); + } + // await this.livekitRoom.localParticipant.enableCameraAndMicrophone(); + } } private stopped = false; @@ -1814,7 +1821,17 @@ export class CallViewModel extends ViewModel { ) { super(); - void this.localConnection.then((c) => void c.startPublishing()); + void from(this.localConnection) + .pipe(this.scope.bind()) + .subscribe( + (c) => + void c + .startPublishing() + // eslint-disable-next-line no-console + .then(() => console.log("successfully started publishing")) + // eslint-disable-next-line no-console + .catch((e) => console.error("failed to start publishing", e)), + ); this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { From a10489d7df451f82bc3c282edc88786f7962fe44 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 17:07:22 +0200 Subject: [PATCH 008/144] publish audio in remote rooms Signed-off-by: Timo K --- src/state/CallViewModel.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index bade6ef9..095b7d03 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -470,6 +470,10 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: { deviceId: "default" }, + }); + await this.livekitRoom.localParticipant.publishTrack(tracks[0]); } public async startPublishing(): Promise { From 6bdfd7fbd8dbf0c2d0a50170e73767c7e00c805f Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 17:08:08 +0200 Subject: [PATCH 009/144] add comment Signed-off-by: Timo K --- src/state/CallViewModel.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 095b7d03..72351042 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -470,6 +470,8 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); + // TODO-MULTI-SFU in this livekit room we really do not want to publish any tracks. + // this is only for testing purposes const tracks = await this.livekitRoom.localParticipant.createTracks({ audio: { deviceId: "default" }, }); From 55b46b3f33ac48f9605d5bd9d44011bbeb398661 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 18:41:03 +0200 Subject: [PATCH 010/144] introduce publishingParticipants$ Signed-off-by: Timo K --- src/state/CallViewModel.ts | 76 +++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 72351042..6c6301ad 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -66,6 +66,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitFocusConfig, + LivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -490,7 +491,6 @@ class Connection { for (const track of tracks) { await this.livekitRoom.localParticipant.publishTrack(track); } - // await this.livekitRoom.localParticipant.enableCameraAndMicrophone(); } } @@ -501,17 +501,45 @@ class Connection { this.stopped = true; } - public readonly participants$ = this.scope.behavior( + public readonly participantsIncludingJustSubscribers$ = this.scope.behavior( connectedParticipantsObserver(this.livekitRoom), [], ); + public readonly publishingParticipants$ = ( + memberships$: Behavior, + ): Observable => + this.scope.behavior( + combineLatest([ + connectedParticipantsObserver(this.livekitRoom), + memberships$, + ]).pipe( + map(([participants, memberships]) => { + const publishingMembers = membershipsFocusUrl( + memberships, + this.matrixRTCSession, + ) + .filter((f) => f.livekit_service_url === this.serviceUrl) + .map((f) => f.membership); + return publishingMembers + .map((m) => + participants.find( + (p) => p.identity === `${m.sender}:${m.deviceId}`, + ), + ) + .filter((p): p is RemoteParticipant => !!p); + }), + ), + [], + ); + public constructor( private readonly livekitRoom: LivekitRoom, private readonly serviceUrl: string, private readonly livekitAlias: string, private readonly client: MatrixClient, private readonly scope: ObservableScope, + private readonly matrixRTCSession: MatrixRTCSession, ) {} } @@ -523,7 +551,7 @@ export class CallViewModel extends ViewModel { private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); - private readonly livekitRoom = new LivekitRoom({ + private readonly localConnectionLivekitRoom = new LivekitRoom({ ...defaultLiveKitOptions, e2ee: this.e2eeOptions, }); @@ -533,11 +561,12 @@ export class CallViewModel extends ViewModel { private readonly localConnection = this.localFocus.then( (focus) => new Connection( - this.livekitRoom, + this.localConnectionLivekitRoom, focus.livekit_service_url, this.livekitAlias, this.matrixRTCSession.room.client, this.scope, + this.matrixRTCSession, ), ); @@ -553,10 +582,9 @@ export class CallViewModel extends ViewModel { map( (memberships) => new Set( - memberships - .map((m) => this.matrixRTCSession.resolveActiveFocus(m)) - .filter((f) => f !== undefined && isLivekitFocusConfig(f)) - .map((f) => f.livekit_service_url), + membershipsFocusUrl(memberships, this.matrixRTCSession).map( + (f) => f.livekit_service_url, + ), ), ), ); @@ -584,6 +612,7 @@ export class CallViewModel extends ViewModel { this.livekitAlias, this.matrixRTCSession.room.client, this.scope, + this.matrixRTCSession, ), ); } @@ -698,7 +727,7 @@ export class CallViewModel extends ViewModel { private readonly remoteParticipants$ = this.scope .behavior< RemoteParticipant[] - >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participants$, ...[...remoteConnections.values()].map((c) => c.participants$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) + >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participantsIncludingJustSubscribers$, ...[...remoteConnections.values()].map((c) => c.participantsIncludingJustSubscribers$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) .pipe(pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( @@ -781,7 +810,7 @@ export class CallViewModel extends ViewModel { private readonly mediaItems$ = this.scope.behavior( combineLatest([ this.remoteParticipants$, - observeParticipantMedia(this.livekitRoom.localParticipant), + observeParticipantMedia(this.localConnectionLivekitRoom.localParticipant), duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -849,7 +878,7 @@ export class CallViewModel extends ViewModel { member, participant, this.options.encryptionSystem, - this.livekitRoom, + this.localConnectionLivekitRoom, this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( @@ -874,7 +903,7 @@ export class CallViewModel extends ViewModel { member, participant, this.options.encryptionSystem, - this.livekitRoom, + this.localConnectionLivekitRoom, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), @@ -916,7 +945,7 @@ export class CallViewModel extends ViewModel { undefined, participant, this.options.encryptionSystem, - this.livekitRoom, + this.localConnectionLivekitRoom, this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( @@ -1862,7 +1891,7 @@ export class CallViewModel extends ViewModel { // that our own media is displayed on screen. this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { const publications = - this.livekitRoom.localParticipant.trackPublications.values(); + this.localConnectionLivekitRoom.localParticipant.trackPublications.values(); if (connected) { for (const p of publications) { if (p.track?.isUpstreamPaused === true) { @@ -1904,3 +1933,22 @@ export class CallViewModel extends ViewModel { this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } } + +const membershipsFocusUrl = ( + memberships: CallMembership[], + matrixRTCSession: MatrixRTCSession, +): { livekit_service_url: string; membership: CallMembership }[] => { + return memberships + .map( + (m) => + [matrixRTCSession.resolveActiveFocus(m), m] as [ + LivekitFocusConfig | undefined, + CallMembership, + ], + ) + .filter(([f, _]) => f !== undefined && isLivekitFocusConfig(f)) + .map(([f, m]) => ({ + livekit_service_url: f!.livekit_service_url, + membership: m, + })); +}; From 8ffb360114cf3d1c82e48b03344901995e3d4d8d Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 10:34:43 +0200 Subject: [PATCH 011/144] add local storage + more readable + remoteParticipants + use publishingParticipants Signed-off-by: Timo K --- src/rtcSessionHelpers.ts | 10 ++++++++++ src/state/CallViewModel.ts | 39 +++++++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index c08fcd40..07cc49fc 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -44,6 +44,16 @@ async function makeFocusInternal( // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); + if (localStorage.getItem("timo-focus-url")) { + const timoFocusUrl = JSON.parse(localStorage.getItem("timo-focus-url")!); + const focusFromUrl: LivekitFocus = { + type: "livekit", + livekit_service_url: timoFocusUrl, + livekit_alias: livekitAlias, + }; + logger.log("Using LiveKit focus from localStorage: ", timoFocusUrl); + return focusFromUrl; + } if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6c6301ad..62f00859 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -66,7 +66,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitFocusConfig, - LivekitFocusConfig, + type LivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -501,7 +501,7 @@ class Connection { this.stopped = true; } - public readonly participantsIncludingJustSubscribers$ = this.scope.behavior( + public readonly participantsIncludingSubscribers$ = this.scope.behavior( connectedParticipantsObserver(this.livekitRoom), [], ); @@ -570,12 +570,14 @@ export class CallViewModel extends ViewModel { ), ); - private readonly memberships$ = fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.memberships), + ), ); private readonly foci$ = this.memberships$.pipe( @@ -725,9 +727,24 @@ export class CallViewModel extends ViewModel { * The RemoteParticipants including those that are being "held" on the screen */ private readonly remoteParticipants$ = this.scope - .behavior< - RemoteParticipant[] - >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participantsIncludingJustSubscribers$, ...[...remoteConnections.values()].map((c) => c.participantsIncludingJustSubscribers$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) + .behavior( + combineLatest( + [this.localConnection, this.remoteConnections$], + (localConnection, remoteConnections) => { + const remoteConnectionsParticipants = [ + ...remoteConnections.values(), + ].map((c) => c.publishingParticipants$(this.memberships$)); + + return combineLatest( + [ + localConnection.publishingParticipants$(this.memberships$), + ...remoteConnectionsParticipants, + ], + (...ps) => ps.flat(1), + ); + }, + ).pipe(switchAll(), startWith([])), + ) .pipe(pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( From 33bc78eec172a0ad5d502f1f93a59e9969f04df9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 11:18:38 +0200 Subject: [PATCH 012/144] add logging Signed-off-by: Timo K --- src/rtcSessionHelpers.ts | 2 +- src/state/CallViewModel.ts | 35 ++++++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 07cc49fc..a88502af 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -45,7 +45,7 @@ async function makeFocusInternal( // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (localStorage.getItem("timo-focus-url")) { - const timoFocusUrl = JSON.parse(localStorage.getItem("timo-focus-url")!); + const timoFocusUrl = localStorage.getItem("timo-focus-url")!; const focusFromUrl: LivekitFocus = { type: "livekit", livekit_service_url: timoFocusUrl, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 62f00859..7c04aafa 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -459,8 +459,8 @@ function getRoomMemberFromRtcMember( return { id, member }; } +// TODO-MULTI-SFU Add all device syncing logic from useLivekit class Connection { - // TODO-MULTI-SFU Add all device syncing logic from useLivekit private readonly sfuConfig = getSFUConfigWithOpenID( this.client, this.serviceUrl, @@ -521,13 +521,34 @@ class Connection { ) .filter((f) => f.livekit_service_url === this.serviceUrl) .map((f) => f.membership); - return publishingMembers - .map((m) => - participants.find( - (p) => p.identity === `${m.sender}:${m.deviceId}`, - ), - ) + + const publishingP = publishingMembers + .map((m) => { + logger.log( + "Publishing participants: all participants at: ", + this.livekitAlias, + this.serviceUrl, + participants, + ); + return participants.find((p) => { + logger.log( + "Publishing participants: compare", + p.identity, + "===", + `${m.sender}:${m.deviceId}`, + ); + return p.identity === `${m.sender}:${m.deviceId}`; + }); + }) .filter((p): p is RemoteParticipant => !!p); + logger.log( + "Publishing participants: find participants for url ", + this.serviceUrl, + publishingMembers, + "Publishing participants: ", + publishingP, + ); + return publishingP; }), ), [], From a617a92e8884af823c76419b308d28e571114ec1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 13:37:17 +0200 Subject: [PATCH 013/144] make it work Signed-off-by: Timo K --- src/state/CallViewModel.ts | 72 ++++++++++++++------------------------ 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7c04aafa..af588e62 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -471,11 +471,6 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); - // TODO-MULTI-SFU in this livekit room we really do not want to publish any tracks. - // this is only for testing purposes - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: { deviceId: "default" }, - }); await this.livekitRoom.localParticipant.publishTrack(tracks[0]); } @@ -483,9 +478,10 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); + if (!this.stopped) { const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: { deviceId: "default" }, + audio: true, video: true, }); for (const track of tracks) { @@ -524,30 +520,11 @@ class Connection { const publishingP = publishingMembers .map((m) => { - logger.log( - "Publishing participants: all participants at: ", - this.livekitAlias, - this.serviceUrl, - participants, - ); return participants.find((p) => { - logger.log( - "Publishing participants: compare", - p.identity, - "===", - `${m.sender}:${m.deviceId}`, - ); return p.identity === `${m.sender}:${m.deviceId}`; }); }) .filter((p): p is RemoteParticipant => !!p); - logger.log( - "Publishing participants: find participants for url ", - this.serviceUrl, - publishingMembers, - "Publishing participants: ", - publishingP, - ); return publishingP; }), ), @@ -612,21 +589,22 @@ export class CallViewModel extends ViewModel { ), ); - private readonly remoteConnections$ = combineLatest([ - this.localFocus, - this.foci$, - ]).pipe( - accumulate(new Map(), (prev, [localFocus, foci]) => { - const stopped = new Map(prev); - const next = new Map(); + private readonly remoteConnections$ = this.scope.behavior( + combineLatest([this.localFocus, this.foci$]).pipe( + accumulate(new Map(), (prev, [localFocus, foci]) => { + const stopped = new Map(prev); + const next = new Map(); + for (const focus of foci) { + if (focus !== localFocus.livekit_service_url) { + stopped.delete(focus); - for (const focus of foci) { - if (focus !== localFocus.livekit_service_url) { - stopped.delete(focus); - next.set( - focus, - prev.get(focus) ?? - new Connection( + let nextConnection = prev.get(focus); + if (!nextConnection) { + logger.log( + "SFU remoteConnections$ construct new connection: ", + focus, + ); + nextConnection = new Connection( new LivekitRoom({ ...defaultLiveKitOptions, e2ee: this.e2eeOptions, @@ -636,14 +614,18 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession.room.client, this.scope, this.matrixRTCSession, - ), - ); + ); + } else { + logger.log("SFU remoteConnections$ use prev connection: ", focus); + } + next.set(focus, nextConnection); + } } - } - for (const connection of stopped.values()) connection.stop(); - return next; - }), + for (const connection of stopped.values()) connection.stop(); + return next; + }), + ), ); private readonly joined$ = new Subject(); From e4a54e3a195ee17b2ef3ddef6245f26b347352a1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 13:52:12 +0200 Subject: [PATCH 014/144] refactor connnection class Signed-off-by: Timo K --- src/state/CallViewModel.ts | 96 ++---------------------------- src/state/Connection.ts | 116 +++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 90 deletions(-) create mode 100644 src/state/Connection.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index af588e62..03463141 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { - connectedParticipantsObserver, observeParticipantEvents, observeParticipantMedia, } from "@livekit/components-core"; @@ -25,7 +24,6 @@ import { type EventTimelineSetHandlerMap, EventType, RoomEvent, - type MatrixClient, RoomStateEvent, SyncState, type Room as MatrixRoom, @@ -117,7 +115,6 @@ import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; import { constant, type Behavior } from "./Behavior"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { defaultLiveKitOptions } from "../livekit/options"; import { enterRTCSession, @@ -126,7 +123,8 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { ECConnectionState } from "../livekit/useECConnectionState"; +import { type ECConnectionState } from "../livekit/useECConnectionState"; +import { Connection, PublishConnection } from "./Connection"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -459,88 +457,6 @@ function getRoomMemberFromRtcMember( return { id, member }; } -// TODO-MULTI-SFU Add all device syncing logic from useLivekit -class Connection { - private readonly sfuConfig = getSFUConfigWithOpenID( - this.client, - this.serviceUrl, - this.livekitAlias, - ); - - public async startSubscribing(): Promise { - this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); - await this.livekitRoom.localParticipant.publishTrack(tracks[0]); - } - - public async startPublishing(): Promise { - this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); - - if (!this.stopped) { - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: true, - video: true, - }); - for (const track of tracks) { - await this.livekitRoom.localParticipant.publishTrack(track); - } - } - } - - private stopped = false; - - public stop(): void { - void this.livekitRoom.disconnect(); - this.stopped = true; - } - - public readonly participantsIncludingSubscribers$ = this.scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], - ); - - public readonly publishingParticipants$ = ( - memberships$: Behavior, - ): Observable => - this.scope.behavior( - combineLatest([ - connectedParticipantsObserver(this.livekitRoom), - memberships$, - ]).pipe( - map(([participants, memberships]) => { - const publishingMembers = membershipsFocusUrl( - memberships, - this.matrixRTCSession, - ) - .filter((f) => f.livekit_service_url === this.serviceUrl) - .map((f) => f.membership); - - const publishingP = publishingMembers - .map((m) => { - return participants.find((p) => { - return p.identity === `${m.sender}:${m.deviceId}`; - }); - }) - .filter((p): p is RemoteParticipant => !!p); - return publishingP; - }), - ), - [], - ); - - public constructor( - private readonly livekitRoom: LivekitRoom, - private readonly serviceUrl: string, - private readonly livekitAlias: string, - private readonly client: MatrixClient, - private readonly scope: ObservableScope, - private readonly matrixRTCSession: MatrixRTCSession, - ) {} -} - export class CallViewModel extends ViewModel { private readonly e2eeOptions = getE2eeOptions( this.options.encryptionSystem, @@ -558,7 +474,7 @@ export class CallViewModel extends ViewModel { private readonly localConnection = this.localFocus.then( (focus) => - new Connection( + new PublishConnection( this.localConnectionLivekitRoom, focus.livekit_service_url, this.livekitAlias, @@ -1881,7 +1797,7 @@ export class CallViewModel extends ViewModel { .subscribe( (c) => void c - .startPublishing() + .start() // eslint-disable-next-line no-console .then(() => console.log("successfully started publishing")) // eslint-disable-next-line no-console @@ -1890,7 +1806,7 @@ export class CallViewModel extends ViewModel { this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { - for (const connection of start) void connection.startSubscribing(); + for (const connection of start) void connection.start(); for (const connection of stop) connection.stop(); }); combineLatest([this.localFocus, this.joined$]) @@ -1954,7 +1870,7 @@ export class CallViewModel extends ViewModel { } } -const membershipsFocusUrl = ( +export const membershipsFocusUrl = ( memberships: CallMembership[], matrixRTCSession: MatrixRTCSession, ): { livekit_service_url: string; membership: CallMembership }[] => { diff --git a/src/state/Connection.ts b/src/state/Connection.ts new file mode 100644 index 00000000..ff5ebb64 --- /dev/null +++ b/src/state/Connection.ts @@ -0,0 +1,116 @@ +// TODO-MULTI-SFU Add all device syncing logic from useLivekit +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { connectedParticipantsObserver } from "@livekit/components-core"; +import { + type Room as LivekitRoom, + type RemoteParticipant, +} from "livekit-client"; +import { type MatrixClient } from "matrix-js-sdk"; +import { + type CallMembership, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, map, type Observable } from "rxjs"; + +import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; +import { type Behavior } from "./Behavior"; +import { membershipsFocusUrl } from "./CallViewModel"; +import { type ObservableScope } from "./ObservableScope"; + +export class Connection { + protected readonly sfuConfig = getSFUConfigWithOpenID( + this.client, + this.serviceUrl, + this.livekitAlias, + ); + + public async start(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + } + + protected stopped = false; + + public stop(): void { + void this.livekitRoom.disconnect(); + this.stopped = true; + } + + public readonly participantsIncludingSubscribers$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); + + public readonly publishingParticipants$ = ( + memberships$: Behavior, + ): Observable => + this.scope.behavior( + combineLatest([ + connectedParticipantsObserver(this.livekitRoom), + memberships$, + ]).pipe( + map(([participants, memberships]) => { + const publishingMembers = membershipsFocusUrl( + memberships, + this.matrixRTCSession, + ) + .filter((f) => f.livekit_service_url === this.serviceUrl) + .map((f) => f.membership); + + const publishingP = publishingMembers + .map((m) => { + return participants.find((p) => { + return p.identity === `${m.sender}:${m.deviceId}`; + }); + }) + .filter((p): p is RemoteParticipant => !!p); + return publishingP; + }), + ), + [], + ); + + public constructor( + protected readonly livekitRoom: LivekitRoom, + protected readonly serviceUrl: string, + protected readonly livekitAlias: string, + protected readonly client: MatrixClient, + protected readonly scope: ObservableScope, + protected readonly matrixRTCSession: MatrixRTCSession, + ) {} +} + +export class PublishConnection extends Connection { + public async start(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + + if (!this.stopped) { + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: true, + video: true, + }); + for (const track of tracks) { + await this.livekitRoom.localParticipant.publishTrack(track); + } + } + } + + public stop(): void { + void this.livekitRoom.disconnect(); + this.stopped = true; + } + + public readonly participantsIncludingSubscribers$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); +} From 802ebf828d5d6f19a00000675442d039df12ac14 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 15:32:46 +0200 Subject: [PATCH 015/144] refactor connection Signed-off-by: Timo K --- src/state/CallViewModel.ts | 101 +++++++++++++++++++++---------------- src/state/Connection.ts | 50 +++++++++--------- 2 files changed, 82 insertions(+), 69 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 03463141..2cbaf738 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -63,6 +63,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, + isLivekitFocus, isLivekitFocusConfig, type LivekitFocusConfig, type MatrixRTCSession, @@ -476,11 +477,11 @@ export class CallViewModel extends ViewModel { (focus) => new PublishConnection( this.localConnectionLivekitRoom, - focus.livekit_service_url, + focus, this.livekitAlias, this.matrixRTCSession.room.client, this.scope, - this.matrixRTCSession, + this.membershipsAndFocusMap$, ), ); @@ -494,53 +495,67 @@ export class CallViewModel extends ViewModel { ), ); - private readonly foci$ = this.memberships$.pipe( - map( - (memberships) => - new Set( - membershipsFocusUrl(memberships, this.matrixRTCSession).map( - (f) => f.livekit_service_url, - ), - ), + private readonly membershipsAndFocusMap$ = this.scope.behavior( + this.memberships$.pipe( + map((memberships) => + memberships.flatMap((m) => { + const f = this.matrixRTCSession.resolveActiveFocus(m); + return f && isLivekitFocus(f) ? [{ membership: m, focus: f }] : []; + }), + ), ), ); + private readonly focusServiceUrls$ = this.membershipsAndFocusMap$.pipe( + map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))), + ); + private readonly remoteConnections$ = this.scope.behavior( - combineLatest([this.localFocus, this.foci$]).pipe( - accumulate(new Map(), (prev, [localFocus, foci]) => { - const stopped = new Map(prev); - const next = new Map(); - for (const focus of foci) { - if (focus !== localFocus.livekit_service_url) { - stopped.delete(focus); + combineLatest([this.localFocus, this.focusServiceUrls$]).pipe( + accumulate( + new Map(), + (prev, [localFocus, focusUrls]) => { + const stopped = new Map(prev); + const next = new Map(); + for (const focusUrl of focusUrls) { + if (focusUrl !== localFocus.livekit_service_url) { + stopped.delete(focusUrl); - let nextConnection = prev.get(focus); - if (!nextConnection) { - logger.log( - "SFU remoteConnections$ construct new connection: ", - focus, - ); - nextConnection = new Connection( - new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: this.e2eeOptions, - }), - focus, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.matrixRTCSession, - ); - } else { - logger.log("SFU remoteConnections$ use prev connection: ", focus); + let nextConnection = prev.get(focusUrl); + if (!nextConnection) { + logger.log( + "SFU remoteConnections$ construct new connection: ", + focusUrl, + ); + nextConnection = new Connection( + new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: this.e2eeOptions, + }), + { + livekit_service_url: focusUrl, + livekit_alias: this.livekitAlias, + type: "livekit", + }, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + this.membershipsAndFocusMap$, + ); + } else { + logger.log( + "SFU remoteConnections$ use prev connection: ", + focusUrl, + ); + } + next.set(focusUrl, nextConnection); } - next.set(focus, nextConnection); } - } - for (const connection of stopped.values()) connection.stop(); - return next; - }), + for (const connection of stopped.values()) connection.stop(); + return next; + }, + ), ), ); @@ -652,11 +667,11 @@ export class CallViewModel extends ViewModel { (localConnection, remoteConnections) => { const remoteConnectionsParticipants = [ ...remoteConnections.values(), - ].map((c) => c.publishingParticipants$(this.memberships$)); + ].map((c) => c.publishingParticipants$); return combineLatest( [ - localConnection.publishingParticipants$(this.memberships$), + localConnection.publishingParticipants$, ...remoteConnectionsParticipants, ], (...ps) => ps.flat(1), diff --git a/src/state/Connection.ts b/src/state/Connection.ts index ff5ebb64..6e114603 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -13,20 +13,19 @@ import { } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { + type LivekitFocus, type CallMembership, - type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, type Observable } from "rxjs"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; -import { membershipsFocusUrl } from "./CallViewModel"; import { type ObservableScope } from "./ObservableScope"; export class Connection { protected readonly sfuConfig = getSFUConfigWithOpenID( this.client, - this.serviceUrl, + this.focus.livekit_service_url, this.livekitAlias, ); @@ -48,42 +47,41 @@ export class Connection { [], ); - public readonly publishingParticipants$ = ( - memberships$: Behavior, - ): Observable => + public readonly publishingParticipants$: Observable = this.scope.behavior( combineLatest([ connectedParticipantsObserver(this.livekitRoom), - memberships$, + this.membershipsFocusMap$, ]).pipe( - map(([participants, memberships]) => { - const publishingMembers = membershipsFocusUrl( - memberships, - this.matrixRTCSession, - ) - .filter((f) => f.livekit_service_url === this.serviceUrl) - .map((f) => f.membership); - - const publishingP = publishingMembers - .map((m) => { - return participants.find((p) => { - return p.identity === `${m.sender}:${m.deviceId}`; - }); - }) - .filter((p): p is RemoteParticipant => !!p); - return publishingP; - }), + map(([participants, membershipsFocusMap]) => + membershipsFocusMap + // Find all members that claim to publish on this connection + .flatMap(({ membership, focus }) => + focus.livekit_service_url === this.focus.livekit_service_url + ? [membership] + : [], + ) + // Find all associated publishing livekit participant objects + .flatMap(({ sender, deviceId }) => { + const participant = participants.find( + (p) => p.identity === `${sender}:${deviceId}`, + ); + return participant ? [participant] : []; + }), + ), ), [], ); public constructor( protected readonly livekitRoom: LivekitRoom, - protected readonly serviceUrl: string, + protected readonly focus: LivekitFocus, protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, - protected readonly matrixRTCSession: MatrixRTCSession, + protected readonly membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, ) {} } From 598371b8077642967732a37d9b656403e9a5dceb Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 17:45:14 +0200 Subject: [PATCH 016/144] lots of work. noone knows if it works. Signed-off-by: Timo K --- src/state/CallViewModel.ts | 391 +++++++++++++++++-------------------- src/state/Connection.ts | 364 +++++++++++++++++++++++++++++++--- 2 files changed, 517 insertions(+), 238 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 2cbaf738..ad6943b9 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -5,15 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - observeParticipantEvents, - observeParticipantMedia, -} from "@livekit/components-core"; +import { observeParticipantEvents } from "@livekit/components-core"; import { ConnectionState, type E2EEOptions, ExternalE2EEKeyProvider, - Room as LivekitRoom, + type Room as LivekitRoom, type LocalParticipant, ParticipantEvent, type RemoteParticipant, @@ -24,10 +21,10 @@ import { type EventTimelineSetHandlerMap, EventType, RoomEvent, + type RoomMember, RoomStateEvent, SyncState, type Room as MatrixRoom, - type RoomMember, } from "matrix-js-sdk"; import { BehaviorSubject, @@ -116,7 +113,7 @@ import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; import { constant, type Behavior } from "./Behavior"; -import { defaultLiveKitOptions } from "../livekit/options"; + import { enterRTCSession, getLivekitAlias, @@ -411,31 +408,6 @@ class ScreenShare { type MediaItem = UserMedia | ScreenShare; -function getE2eeOptions( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): E2EEOptions | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return { - keyProvider, - worker: new E2EEWorker(), - }; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return { - keyProvider, - worker: new E2EEWorker(), - }; - } -} - function getRoomMemberFromRtcMember( rtcMember: CallMembership, room: MatrixRoom, @@ -459,29 +431,25 @@ function getRoomMemberFromRtcMember( } export class CallViewModel extends ViewModel { - private readonly e2eeOptions = getE2eeOptions( + private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); + + private readonly livekitE2EERoomOptions = getE2eeOptions( this.options.encryptionSystem, this.matrixRTCSession, ); - private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); - - private readonly localConnectionLivekitRoom = new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: this.e2eeOptions, - }); - private readonly localFocus = makeFocus(this.matrixRTCSession); private readonly localConnection = this.localFocus.then( (focus) => new PublishConnection( - this.localConnectionLivekitRoom, focus, this.livekitAlias, this.matrixRTCSession.room.client, this.scope, this.membershipsAndFocusMap$, + this.mediaDevices, + this.livekitE2EERoomOptions, ), ); @@ -506,12 +474,12 @@ export class CallViewModel extends ViewModel { ), ); - private readonly focusServiceUrls$ = this.membershipsAndFocusMap$.pipe( + private readonly livekitServiceUrls$ = this.membershipsAndFocusMap$.pipe( map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))), ); private readonly remoteConnections$ = this.scope.behavior( - combineLatest([this.localFocus, this.focusServiceUrls$]).pipe( + combineLatest([this.localFocus, this.livekitServiceUrls$]).pipe( accumulate( new Map(), (prev, [localFocus, focusUrls]) => { @@ -528,10 +496,6 @@ export class CallViewModel extends ViewModel { focusUrl, ); nextConnection = new Connection( - new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: this.e2eeOptions, - }), { livekit_service_url: focusUrl, livekit_alias: this.livekitAlias, @@ -541,6 +505,7 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession.room.client, this.scope, this.membershipsAndFocusMap$, + this.livekitE2EERoomOptions, ); } else { logger.log( @@ -657,29 +622,54 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - /** - * The RemoteParticipants including those that are being "held" on the screen - */ - private readonly remoteParticipants$ = this.scope - .behavior( - combineLatest( - [this.localConnection, this.remoteConnections$], - (localConnection, remoteConnections) => { - const remoteConnectionsParticipants = [ - ...remoteConnections.values(), - ].map((c) => c.publishingParticipants$); - - return combineLatest( - [ - localConnection.publishingParticipants$, - ...remoteConnectionsParticipants, - ], - (...ps) => ps.flat(1), + private readonly participants$ = this.scope + .behavior< + { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + livekitRoom: LivekitRoom; + }[] + >( + from(this.localConnection).pipe( + switchMap((localConnection) => { + const memberError = (): never => { + throw new Error("No room member for call membership"); + }; + const localParticipant = { + participant: localConnection.livekitRoom.localParticipant, + member: + this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), + livekitRoom: localConnection.livekitRoom, + }; + return this.remoteConnections$.pipe( + switchMap((connections) => + combineLatest( + [...connections.values()].map((c) => + c.publishingParticipants$.pipe( + map((ps) => + ps.map(({ participant, membership }) => ({ + participant, + member: + getRoomMemberFromRtcMember( + membership, + this.matrixRoom, + )?.member ?? memberError(), + livekitRoom: c.livekitRoom, + })), + ), + ), + ), + ), + ), + map((remoteParticipants) => [ + ...remoteParticipants.flat(1), + localParticipant, + ]), ); - }, - ).pipe(switchAll(), startWith([])), + }), + ), ) - .pipe(pauseWhen(this.pretendToBeDisconnected$)); + .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( fromEvent( @@ -760,8 +750,7 @@ export class CallViewModel extends ViewModel { */ private readonly mediaItems$ = this.scope.behavior( combineLatest([ - this.remoteParticipants$, - observeParticipantMedia(this.localConnectionLivekitRoom.localParticipant), + this.participants$, duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -769,44 +758,17 @@ export class CallViewModel extends ViewModel { scan( ( prevItems, - [ - remoteParticipants, - { participant: localParticipant }, - duplicateTiles, - memberships, - showNonMemberTiles, - ], + [participants, duplicateTiles, memberships, showNonMemberTiles], ) => { - const newItems = new Map( + const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - const room = this.matrixRoom; - // m.rtc.members are the basis for calculating what is visible in the call - for (const rtcMember of memberships) { - const { member, id: livekitParticipantId } = - getRoomMemberFromRtcMember(rtcMember, room); - const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; - - let participant: - | LocalParticipant - | RemoteParticipant - | undefined = undefined; - if (livekitParticipantId === "local") { - participant = localParticipant; - } else { - participant = remoteParticipants.find( - (p) => p.identity === livekitParticipantId, - ); - } - - if (!member) { - logger.error( - "Could not find member for media id: ", - livekitParticipantId, - ); - } + for (const { participant, member, livekitRoom } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; for (let i = 0; i < 1 + duplicateTiles; i++) { - const indexedMediaId = `${livekitParticipantId}:${i}`; - let prevMedia = prevItems.get(indexedMediaId); + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); if (prevMedia && prevMedia instanceof UserMedia) { prevMedia.updateParticipant(participant); if (prevMedia.vm.member === undefined) { @@ -819,33 +781,33 @@ export class CallViewModel extends ViewModel { } } yield [ - indexedMediaId, + mediaId, // We create UserMedia with or without a participant. // This will be the initial value of a BehaviourSubject. // Once a participant appears we will update the BehaviourSubject. (see above) prevMedia ?? new UserMedia( - indexedMediaId, + mediaId, member, participant, this.options.encryptionSystem, - this.localConnectionLivekitRoom, + livekitRoom, this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixIdentifier) ?? "[👻]"), + map((m) => m.get(matrixId) ?? "[👻]"), ), this.handsRaised$.pipe( - map((v) => v[matrixIdentifier]?.time ?? null), + map((v) => v[matrixId]?.time ?? null), ), this.reactions$.pipe( - map((v) => v[matrixIdentifier] ?? undefined), + map((v) => v[matrixId] ?? undefined), ), ), ]; if (participant?.isScreenShareEnabled) { - const screenShareId = `${indexedMediaId}:screen-share`; + const screenShareId = `${mediaId}:screen-share`; yield [ screenShareId, prevItems.get(screenShareId) ?? @@ -854,10 +816,10 @@ export class CallViewModel extends ViewModel { member, participant, this.options.encryptionSystem, - this.localConnectionLivekitRoom, + livekitRoom, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixIdentifier) ?? "[👻]"), + map((m) => m.get(matrixId) ?? "[👻]"), ), ), ]; @@ -879,47 +841,51 @@ export class CallViewModel extends ViewModel { // - If one wants to test scalability using the LiveKit CLI. // - If an experimental project does not yet do the MatrixRTC bits. // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. - const newNonMemberItems = showNonMemberTiles - ? new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const participant of remoteParticipants) { - for (let i = 0; i < 1 + duplicateTiles; i++) { - const maybeNonMemberParticipantId = - participant.identity + ":" + i; - if (!newItems.has(maybeNonMemberParticipantId)) { - const nonMemberId = maybeNonMemberParticipantId; - yield [ - nonMemberId, - prevItems.get(nonMemberId) ?? - new UserMedia( - nonMemberId, - undefined, - participant, - this.options.encryptionSystem, - this.localConnectionLivekitRoom, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map( - (m) => m.get(participant.identity) ?? "[👻]", - ), - ), - of(null), - of(null), - ), - ]; - } - } - } - }.bind(this)(), - ) - : new Map(); - if (newNonMemberItems.size > 0) { - logger.debug("Added NonMember items: ", newNonMemberItems); - } + // TODO-MULTI-SFU + // const newNonMemberItems = showNonMemberTiles + // ? new Map( + // function* ( + // this: CallViewModel, + // ): Iterable<[string, MediaItem]> { + // for (const participant of remoteParticipants) { + // for (let i = 0; i < 1 + duplicateTiles; i++) { + // const maybeNonMemberParticipantId = + // participant.identity + ":" + i; + // if (!newItems.has(maybeNonMemberParticipantId)) { + // const nonMemberId = maybeNonMemberParticipantId; + // yield [ + // nonMemberId, + // prevItems.get(nonMemberId) ?? + // new UserMedia( + // nonMemberId, + // undefined, + // participant, + // this.options.encryptionSystem, + // localConnection.livekitRoom, + // this.mediaDevices, + // this.pretendToBeDisconnected$, + // this.memberDisplaynames$.pipe( + // map( + // (m) => + // m.get(participant.identity) ?? "[👻]", + // ), + // ), + // of(null), + // of(null), + // ), + // ]; + // } + // } + // } + // }.bind(this)(), + // ) + // : new Map(); + // if (newNonMemberItems.size > 0) { + // logger.debug("Added NonMember items: ", newNonMemberItems); + // } const combinedNew = new Map([ - ...newNonMemberItems.entries(), + // ...newNonMemberItems.entries(), ...newItems.entries(), ]); @@ -1840,66 +1806,77 @@ export class CallViewModel extends ViewModel { // We use matrixConnected$ rather than reconnecting$ because we want to // pause tracks during the initial joining sequence too until we're sure // that our own media is displayed on screen. - this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { - const publications = - this.localConnectionLivekitRoom.localParticipant.trackPublications.values(); - if (connected) { - for (const p of publications) { - if (p.track?.isUpstreamPaused === true) { - const kind = p.track.kind; - logger.log( - `Resumming ${kind} track (MatrixRTC connection present)`, - ); - p.track - .resumeUpstream() - .catch((e) => - logger.error( - `Failed to resume ${kind} track after MatrixRTC reconnection`, - e, - ), + void this.localConnection.then((localConnection) => + this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { + const publications = + localConnection.livekitRoom.localParticipant.trackPublications.values(); + if (connected) { + for (const p of publications) { + if (p.track?.isUpstreamPaused === true) { + const kind = p.track.kind; + logger.log( + `Resuming ${kind} track (MatrixRTC connection present)`, ); + p.track + .resumeUpstream() + .catch((e) => + logger.error( + `Failed to resume ${kind} track after MatrixRTC reconnection`, + e, + ), + ); + } + } + } else { + for (const p of publications) { + if (p.track?.isUpstreamPaused === false) { + const kind = p.track.kind; + logger.log( + `Pausing ${kind} track (uncertain MatrixRTC connection)`, + ); + p.track + .pauseUpstream() + .catch((e) => + logger.error( + `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, + e, + ), + ); + } } } - } else { - for (const p of publications) { - if (p.track?.isUpstreamPaused === false) { - const kind = p.track.kind; - logger.log( - `Pausing ${kind} track (uncertain MatrixRTC connection)`, - ); - p.track - .pauseUpstream() - .catch((e) => - logger.error( - `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, - e, - ), - ); - } - } - } - }); + }), + ); // Join automatically this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } } -export const membershipsFocusUrl = ( - memberships: CallMembership[], - matrixRTCSession: MatrixRTCSession, -): { livekit_service_url: string; membership: CallMembership }[] => { - return memberships - .map( - (m) => - [matrixRTCSession.resolveActiveFocus(m), m] as [ - LivekitFocusConfig | undefined, - CallMembership, - ], - ) - .filter(([f, _]) => f !== undefined && isLivekitFocusConfig(f)) - .map(([f, m]) => ({ - livekit_service_url: f!.livekit_service_url, - membership: m, - })); -}; +// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes +// do we need this? + +function getE2eeOptions( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): E2EEOptions | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } +} diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 6e114603..700ee4ef 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -8,26 +8,42 @@ Please see LICENSE in the repository root for full details. import { connectedParticipantsObserver } from "@livekit/components-core"; import { - type Room as LivekitRoom, - type RemoteParticipant, + ConnectionState, + Room as LivekitRoom, + type RoomOptions, + type E2EEOptions, + RoomEvent, + Track, } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { type LivekitFocus, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, type Observable } from "rxjs"; +import { + BehaviorSubject, + combineLatest, + filter, + fromEvent, + map, + NEVER, + type Observable, + type Subscription, + switchMap, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type SelectedDevice, type MediaDevices } from "./MediaDevices"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; -import { type Behavior } from "./Behavior"; +import { constant, type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; +import { defaultLiveKitOptions } from "../livekit/options"; +import { getValue } from "../utils/observable"; +import { getUrlParams } from "../UrlParams"; +import { type MuteStates } from "../room/MuteStates"; export class Connection { - protected readonly sfuConfig = getSFUConfigWithOpenID( - this.client, - this.focus.livekit_service_url, - this.livekitAlias, - ); + protected stopped = false; public async start(): Promise { this.stopped = false; @@ -35,22 +51,44 @@ export class Connection { if (!this.stopped) await this.livekitRoom.connect(url, jwt); } - protected stopped = false; - public stop(): void { void this.livekitRoom.disconnect(); this.stopped = true; } - public readonly participantsIncludingSubscribers$ = this.scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], + protected readonly sfuConfig = getSFUConfigWithOpenID( + this.client, + this.focus.livekit_service_url, + this.livekitAlias, ); - public readonly publishingParticipants$: Observable = - this.scope.behavior( + public readonly participantsIncludingSubscribers$; + public readonly publishingParticipants$; + public livekitRoom: LivekitRoom; + + public connectionState$: Behavior; + public constructor( + protected readonly focus: LivekitFocus, + protected readonly livekitAlias: string, + protected readonly client: MatrixClient, + protected readonly scope: ObservableScope, + protected readonly membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, + e2eeLivekitOptions: E2EEOptions | undefined, + ) { + this.livekitRoom = new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: e2eeLivekitOptions, + }); + this.participantsIncludingSubscribers$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); + + this.publishingParticipants$ = this.scope.behavior( combineLatest([ - connectedParticipantsObserver(this.livekitRoom), + this.participantsIncludingSubscribers$, this.membershipsFocusMap$, ]).pipe( map(([participants, membershipsFocusMap]) => @@ -62,27 +100,24 @@ export class Connection { : [], ) // Find all associated publishing livekit participant objects - .flatMap(({ sender, deviceId }) => { + .flatMap((membership) => { const participant = participants.find( - (p) => p.identity === `${sender}:${deviceId}`, + (p) => + p.identity === `${membership.sender}:${membership.deviceId}`, ); - return participant ? [participant] : []; + return participant ? [{ participant, membership }] : []; }), ), ), [], ); - - public constructor( - protected readonly livekitRoom: LivekitRoom, - protected readonly focus: LivekitFocus, - protected readonly livekitAlias: string, - protected readonly client: MatrixClient, - protected readonly scope: ObservableScope, - protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, - ) {} + this.connectionState$ = this.scope.behavior( + fromEvent( + this.livekitRoom, + RoomEvent.ConnectionStateChanged, + ), + ); + } } export class PublishConnection extends Connection { @@ -111,4 +146,271 @@ export class PublishConnection extends Connection { connectedParticipantsObserver(this.livekitRoom), [], ); + private readonly muteStates$: Behavior; + private updatingMuteStates$ = new BehaviorSubject(false); + + public constructor( + protected readonly focus: LivekitFocus, + protected readonly livekitAlias: string, + protected readonly client: MatrixClient, + protected readonly scope: ObservableScope, + protected readonly membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, + protected readonly devices: MediaDevices, + e2eeLivekitOptions: E2EEOptions | undefined, + ) { + super( + focus, + livekitAlias, + client, + scope, + membershipsFocusMap$, + e2eeLivekitOptions, + ); + + // TODO-MULTI-SFU use actual mute states + this.muteStates$ = constant({ + audio: { enabled: true, setEnabled: (enabled) => {} }, + video: { enabled: true, setEnabled: (enabled) => {} }, + }); + + logger.info("[LivekitRoom] Create LiveKit room"); + const { controlledAudioDevices } = getUrlParams(); + + const roomOptions: RoomOptions = { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: getValue(this.devices.videoInput.selected$)?.id, + // TODO-MULTI-SFU add processor support back + // processor, + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: getValue(devices.audioInput.selected$)?.id, + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id, + }, + e2ee: e2eeLivekitOptions, + }; + // We have to create the room manually here due to a bug inside + // @livekit/components-react. JSON.stringify() is used in deps of a + // useEffect() with an argument that references itself, if E2EE is enabled + const room = new LivekitRoom(roomOptions); + room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + this.livekitRoom = room; + + // sync mute states TODO-MULTI_SFU This possibly can be simplified quite a bit. + combineLatest([ + this.connectionState$, + this.muteStates$, + this.updatingMuteStates$, + ]) + .pipe( + filter(([_c, _m, updating]) => !updating), + this.scope.bind(), + ) + .subscribe(([connectionState, muteStates, _]) => { + // Sync the requested mute states with LiveKit's mute states. We do it this + // way around rather than using LiveKit as the source of truth, so that the + // states can be consistent throughout the lobby and loading screens. + // It's important that we only do this in the connected state, because + // LiveKit's internal mute states aren't consistent during connection setup, + // and setting tracks to be enabled during this time causes errors. + if ( + this.livekitRoom !== undefined && + connectionState === ConnectionState.Connected + ) { + const participant = this.livekitRoom.localParticipant; + + enum MuteDevice { + Microphone, + Camera, + } + + const syncMuteState = async ( + iterCount: number, + type: MuteDevice, + ): Promise => { + // The approach for muting is to always bring the actual livekit state in sync with the button + // This allows for a very predictable and reactive behavior for the user. + // (the new state is the old state when pressing the button n times (where n is even)) + // (the new state is different to the old state when pressing the button n times (where n is uneven)) + // In case there are issues with the device there might be situations where setMicrophoneEnabled/setCameraEnabled + // return immediately. This should be caught with the Error("track with new mute state could not be published"). + // For now we are still using an iterCount to limit the recursion loop to 10. + // This could happen if the device just really does not want to turn on (hardware based issue) + // but the mute button is in unmute state. + // For now our fail mode is to just stay in this state. + // TODO: decide for a UX on how that fail mode should be treated (disable button, hide button, sync button back to muted without user input) + + if (iterCount > 10) { + logger.error( + "Stop trying to sync the input device with current mute state after 10 failed tries", + ); + return; + } + let devEnabled; + let btnEnabled; + switch (type) { + case MuteDevice.Microphone: + devEnabled = participant.isMicrophoneEnabled; + btnEnabled = muteStates.audio.enabled; + break; + case MuteDevice.Camera: + devEnabled = participant.isCameraEnabled; + btnEnabled = muteStates.video.enabled; + break; + } + if (devEnabled !== btnEnabled && !this.updatingMuteStates$.value) { + this.updatingMuteStates$.next(true); + + try { + let trackPublication; + switch (type) { + case MuteDevice.Microphone: + trackPublication = await participant.setMicrophoneEnabled( + btnEnabled, + this.livekitRoom.options.audioCaptureDefaults, + ); + break; + case MuteDevice.Camera: + trackPublication = await participant.setCameraEnabled( + btnEnabled, + this.livekitRoom.options.videoCaptureDefaults, + ); + break; + } + + if (trackPublication) { + // await participant.setMicrophoneEnabled can return immediately in some instances, + // so that participant.isMicrophoneEnabled !== buttonEnabled.current.audio still holds true. + // This happens if the device is still in a pending state + // "sleeping" here makes sure we let react do its thing so that participant.isMicrophoneEnabled is updated, + // so we do not end up in a recursion loop. + await new Promise((r) => setTimeout(r, 100)); + + // track got successfully changed to mute/unmute + // Run the check again after the change is done. Because the user + // can update the state (presses mute button) while the device is enabling + // itself we need might need to update the mute state right away. + // This async recursion makes sure that setCamera/MicrophoneEnabled is + // called as little times as possible. + await syncMuteState(iterCount + 1, type); + } else { + throw new Error( + "track with new mute state could not be published", + ); + } + } catch (e) { + if ((e as DOMException).name === "NotAllowedError") { + logger.error( + "Fatal error while syncing mute state: resetting", + e, + ); + if (type === MuteDevice.Microphone) { + muteStates.audio.setEnabled?.(false); + } else { + muteStates.video.setEnabled?.(false); + } + } else { + logger.error( + "Failed to sync audio mute state with LiveKit (will retry to sync in 1s):", + e, + ); + setTimeout(() => { + this.updatingMuteStates$.next(false); + }, 1000); + } + } + } + }; + + syncMuteState(0, MuteDevice.Microphone).catch((e) => { + logger.error("Failed to sync audio mute state with LiveKit", e); + }); + syncMuteState(0, MuteDevice.Camera).catch((e) => { + logger.error("Failed to sync video mute state with LiveKit", e); + }); + } + }); + + const syncDevice = ( + kind: MediaDeviceKind, + selected$: Observable, + ): Subscription => + selected$.pipe(this.scope.bind()).subscribe((device) => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + logger.info( + "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", + this.livekitRoom.getActiveDevice(kind), + " !== ", + device?.id, + ); + if ( + device !== undefined && + this.livekitRoom.getActiveDevice(kind) !== device.id + ) { + this.livekitRoom + .switchActiveDevice(kind, device.id) + .catch((e) => + logger.error(`Failed to sync ${kind} device with LiveKit`, e), + ); + } + }); + + syncDevice("audioinput", devices.audioInput.selected$); + if (!controlledAudioDevices) + syncDevice("audiooutput", devices.audioOutput.selected$); + syncDevice("videoinput", devices.videoInput.selected$); + // Restart the audio input track whenever we detect that the active media + // device has changed to refer to a different hardware device. We do this + // for the sake of Chrome, which provides a "default" device that is meant + // to match the system's default audio input, whatever that may be. + // This is special-cased for only audio inputs because we need to dig around + // in the LocalParticipant object for the track object and there's not a nice + // way to do that generically. There is usually no OS-level default video capture + // device anyway, and audio outputs work differently. + devices.audioInput.selected$ + .pipe( + switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), + this.scope.bind(), + ) + .subscribe(() => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + const activeMicTrack = Array.from( + this.livekitRoom.localParticipant.audioTrackPublications.values(), + ).find((d) => d.source === Track.Source.Microphone)?.track; + + if ( + activeMicTrack && + // only restart if the stream is still running: LiveKit will detect + // when a track stops & restart appropriately, so this is not our job. + // Plus, we need to avoid restarting again if the track is already in + // the process of being restarted. + activeMicTrack.mediaStreamTrack.readyState !== "ended" + ) { + // Restart the track, which will cause Livekit to do another + // getUserMedia() call with deviceId: default to get the *new* default device. + // Note that room.switchActiveDevice() won't work: Livekit will ignore it because + // the deviceId hasn't changed (was & still is default). + this.livekitRoom.localParticipant + .getTrackPublication(Track.Source.Microphone) + ?.audioTrack?.restartTrack() + .catch((e) => { + logger.error(`Failed to restart audio device track`, e); + }); + } + }); + } + // TODO-MULTI-SFU Sync the requested track processors with LiveKit } From 02f4c73759b686482bc13222b077f3cbfae1b9dc Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 28 Aug 2025 11:02:41 +0200 Subject: [PATCH 017/144] Add my own local storage SFU config stuff too --- src/rtcSessionHelpers.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index a88502af..38d9e849 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -42,6 +42,17 @@ async function makeFocusInternal( logger.log("Searching for a preferred focus"); const livekitAlias = getLivekitAlias(rtcSession); + const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth"); + if (urlFromStorage !== null) { + const focusFromStorage: LivekitFocus = { + type: "livekit", + livekit_service_url: urlFromStorage, + livekit_alias: livekitAlias, + }; + logger.log("Using LiveKit focus from local storage: ", focusFromStorage); + return focusFromStorage; + } + // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (localStorage.getItem("timo-focus-url")) { From d46fe55a670190a58822312725de3d08ce12e0df Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 28 Aug 2025 17:40:35 +0200 Subject: [PATCH 018/144] Import unfinished mute states refactor --- src/room/GroupCallView.tsx | 8 +- src/room/RoomPage.tsx | 26 ++++-- src/state/MuteStates.ts | 163 +++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 src/state/MuteStates.ts diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 8562d4f8..40ec4627 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -38,7 +38,7 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useProfile } from "../profile/useProfile"; import { findDeviceByName } from "../utils/media"; import { ActiveCall } from "./InCallView"; -import { MUTE_PARTICIPANT_COUNT, type MuteStates } from "./MuteStates"; +import { type MuteStates } from "../state/MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { leaveRTCSession } from "../rtcSessionHelpers"; @@ -76,6 +76,12 @@ import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useAppBarTitle } from "../AppBar.tsx"; import { useBehavior } from "../useBehavior.ts"; +/** + * If there already are this many participants in the call, we automatically mute + * the user. + */ +export const MUTE_PARTICIPANT_COUNT = 8; + declare global { interface Window { rtcSession?: MatrixRTCSession; diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 480f8706..3924437b 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -20,6 +20,8 @@ import { CheckIcon, UnknownSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; +import { useObservable } from "observable-hooks"; +import { map } from "rxjs"; import { useClientLegacy } from "../ClientContext"; import { ErrorPage, FullScreenView, LoadingPage } from "../FullScreenView"; @@ -35,12 +37,13 @@ import { CallTerminatedMessage, useLoadGroupCall } from "./useLoadGroupCall"; import { LobbyView } from "./LobbyView"; import { E2eeType } from "../e2ee/e2eeType"; import { useProfile } from "../profile/useProfile"; -import { useMuteStates } from "./MuteStates"; import { useOptInAnalytics } from "../settings/settings"; import { Config } from "../config/Config"; import { Link } from "../button/Link"; import { ErrorView } from "../ErrorView"; -import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState"; +import { useMediaDevices } from "../MediaDevicesContext"; +import { MuteStates } from "../state/MuteStates"; +import { ObservableScope } from "../state/ObservableScope"; export const RoomPage: FC = () => { const { confineToRoom, appPrompt, preload, header, displayName, skipLobby } = @@ -62,7 +65,18 @@ export const RoomPage: FC = () => { const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers); const [joined, setJoined] = useState(false); - const muteStates = useMuteStates(joined); + + const devices = useMediaDevices(); + const [muteStates, setMuteStates] = useState(null); + const joined$ = useObservable( + (inputs$) => inputs$.pipe(map(([joined]) => joined)), + [joined], + ); + useEffect(() => { + const scope = new ObservableScope(); + setMuteStates(new MuteStates(scope, devices, joined$)); + return (): void => scope.end(); + }, [devices, joined$]); useEffect(() => { // If we've finished loading, are not already authed and we've been given a display name as @@ -99,10 +113,10 @@ export const RoomPage: FC = () => { } }, [groupCallState.kind]); - const groupCallView = (): JSX.Element => { + const groupCallView = (): ReactNode => { switch (groupCallState.kind) { case "loaded": - return ( + return muteStates && ( { ); return ( - ; + set: ((enabled: boolean) => void) | null; + toggle: (() => void) | null; +} + +class MuteState { + private readonly enabledByDefault$ = + this.enabledByConfig && !getUrlParams().skipLobby + ? this.isJoined$.pipe(map((isJoined) => !isJoined)) + : of(false); + + private readonly data$: Observable = + this.device.available$.pipe( + map((available) => available.size > 0), + distinctUntilChanged(), + withLatestFrom( + this.enabledByDefault$, + (devicesConnected, enabledByDefault) => { + if (!devicesConnected) + return { enabled$: of(false), set: null, toggle: null }; + + const set$ = new Subject(); + const toggle$ = new Subject(); + return { + set: (enabled: boolean) => set$.next(enabled), + toggle: () => toggle$.next(), + // Assume the default value only once devices are actually connected + enabled$: merge( + set$, + toggle$.pipe(map(() => "toggle" as const)), + ).pipe( + accumulate(enabledByDefault, (prev, update) => + update === "toggle" ? !prev : update, + ), + ), + }; + }, + ), + this.scope.state(), + ); + + public readonly enabled$: Observable = this.data$.pipe( + switchMap(({ enabled$ }) => enabled$), + ); + + public readonly setEnabled$: Observable<((enabled: boolean) => void) | null> = + this.data$.pipe(map(({ set }) => set)); + + public readonly toggle$: Observable<(() => void) | null> = this.data$.pipe( + map(({ toggle }) => toggle), + ); + + public constructor( + private readonly scope: ObservableScope, + private readonly device: MediaDevice, + private readonly isJoined$: Observable, + private readonly enabledByConfig: boolean, + ) {} +} + +export class MuteStates { + public readonly audio = new MuteState( + this.scope, + this.mediaDevices.audioInput, + this.isJoined$, + Config.get().media_devices.enable_video, + ); + public readonly video = new MuteState( + this.scope, + this.mediaDevices.videoInput, + this.isJoined$, + Config.get().media_devices.enable_video, + ); + + public constructor( + private readonly scope: ObservableScope, + private readonly mediaDevices: MediaDevices, + private readonly isJoined$: Observable, + ) { + if (widget !== null) { + // Sync our mute states with the hosting client + const widgetApiState$ = combineLatest( + [this.audio.enabled$, this.video.enabled$], + (audio, video) => ({ audio_enabled: audio, video_enabled: video }), + ); + widgetApiState$.pipe(this.scope.bind()).subscribe((state) => { + widget!.api.transport + .send(ElementWidgetActions.DeviceMute, state) + .catch((e) => + logger.warn("Could not send DeviceMute action to widget", e), + ); + }); + + // Also sync the hosting client's mute states back with ours + const muteActions$ = fromEvent( + widget.lazyActions, + ElementWidgetActions.DeviceMute, + ) as Observable>; + muteActions$ + .pipe( + withLatestFrom( + widgetApiState$, + this.audio.setEnabled$, + this.video.setEnabled$, + ), + this.scope.bind(), + ) + .subscribe(([ev, state, setAudioEnabled, setVideoEnabled]) => { + // First copy the current state into our new state + const newState = { ...state }; + // Update new state if there are any requested changes from the widget + // action in `ev.detail.data`. + if ( + ev.detail.data.audio_enabled != null && + typeof ev.detail.data.audio_enabled === "boolean" && + setAudioEnabled !== null + ) { + newState.audio_enabled = ev.detail.data.audio_enabled; + setAudioEnabled(newState.audio_enabled); + } + if ( + ev.detail.data.video_enabled != null && + typeof ev.detail.data.video_enabled === "boolean" && + setVideoEnabled !== null + ) { + newState.video_enabled = ev.detail.data.video_enabled; + setVideoEnabled(newState.video_enabled); + } + widget!.api.transport.reply(ev.detail, newState); + }); + } + } +} From 386dc6c84d8db8ec751e283901f52837e6089f7b Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 18:41:13 +0200 Subject: [PATCH 019/144] temp before holiday Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 6 ++-- src/state/CallViewModel.ts | 67 +++++++++++++++++++------------------- src/state/MuteStates.ts | 39 +++++++++++----------- 3 files changed, 57 insertions(+), 55 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 40ec4627..b2495f54 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -132,7 +132,7 @@ export const GroupCallView: FC = ({ // This should use `useEffectEvent` (only available in experimental versions) useEffect(() => { if (memberships.length >= MUTE_PARTICIPANT_COUNT) - muteStates.audio.setEnabled?.(false); + muteStates.audio.setEnabled$.value?.(false); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); @@ -261,7 +261,7 @@ export const GroupCallView: FC = ({ if (!deviceId) { logger.warn("Unknown audio input: " + audioInput); // override the default mute state - latestMuteStates.current!.audio.setEnabled?.(false); + latestMuteStates.current!.audio.setEnabled$.value?.(false); } else { logger.debug( `Found audio input ID ${deviceId} for name ${audioInput}`, @@ -275,7 +275,7 @@ export const GroupCallView: FC = ({ if (!deviceId) { logger.warn("Unknown video input: " + videoInput); // override the default mute state - latestMuteStates.current!.video.setEnabled?.(false); + latestMuteStates.current!.video.setEnabled$.value?.(false); } else { logger.debug( `Found video input ID ${deviceId} for name ${videoInput}`, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index ad6943b9..b5112e5e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,13 +18,13 @@ import { import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { ClientEvent, - type EventTimelineSetHandlerMap, - EventType, - RoomEvent, type RoomMember, RoomStateEvent, SyncState, type Room as MatrixRoom, + type EventTimelineSetHandlerMap, + EventType, + RoomEvent, } from "matrix-js-sdk"; import { BehaviorSubject, @@ -61,8 +61,6 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitFocus, - isLivekitFocusConfig, - type LivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -112,11 +110,11 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { constant, type Behavior } from "./Behavior"; - +import { type Behavior } from "./Behavior"; import { enterRTCSession, getLivekitAlias, + leaveRTCSession, makeFocus, } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; @@ -453,16 +451,6 @@ export class CallViewModel extends ViewModel { ), ); - private readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), - ), - ); - private readonly membershipsAndFocusMap$ = this.scope.behavior( this.memberships$.pipe( map((memberships) => @@ -524,17 +512,19 @@ export class CallViewModel extends ViewModel { ), ); - private readonly joined$ = new Subject(); + private readonly join$ = new Subject(); public join(): void { - this.joined$.next(); + this.join$.next(); } + private readonly leave$ = new Subject(); + public leave(): void { - // TODO + this.leave$.next(); } - private readonly connectionInstructions$ = this.joined$.pipe( + private readonly connectionInstructions$ = this.join$.pipe( switchMap(() => this.remoteConnections$), startWith(new Map()), pairwise(), @@ -622,6 +612,17 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + pauseWhen(this.pretendToBeDisconnected$), + map(() => this.matrixRTCSession.memberships), + ), + ); + private readonly participants$ = this.scope .behavior< { @@ -671,17 +672,6 @@ export class CallViewModel extends ViewModel { ) .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); - private readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - pauseWhen(this.pretendToBeDisconnected$), - map(() => this.matrixRTCSession.memberships), - ), - ); - /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members @@ -1790,7 +1780,7 @@ export class CallViewModel extends ViewModel { for (const connection of start) void connection.start(); for (const connection of stop) connection.stop(); }); - combineLatest([this.localFocus, this.joined$]) + combineLatest([this.localFocus, this.join$]) .pipe(this.scope.bind()) .subscribe(([localFocus]) => { void enterRTCSession( @@ -1799,6 +1789,17 @@ export class CallViewModel extends ViewModel { this.options.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, ); }); + this.join$.pipe(this.scope.bind()).subscribe(() => { + leaveRTCSession( + this.matrixRTCSession, + "user", // TODO-MULTI-SFU ? + // Wait for the sound in widget mode (it's not long) + Promise.resolve(), // TODO-MULTI-SFU + //Promise.all([audioPromise, posthogRequest]), + ).catch((e) => { + logger.error("Error leaving RTC session", e); + }); + }); // Pause upstream of all local media tracks when we're disconnected from // MatrixRTC, because it can be an unpleasant surprise for the app to say diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index d5425163..4ac095b0 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -26,6 +26,7 @@ import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; import { type ObservableScope } from "./ObservableScope"; import { accumulate } from "../utils/observable"; +import { type Behavior } from "./Behavior"; interface MuteStateData { enabled$: Observable; @@ -33,13 +34,13 @@ interface MuteStateData { toggle: (() => void) | null; } -class MuteState { +class MuteState { private readonly enabledByDefault$ = this.enabledByConfig && !getUrlParams().skipLobby - ? this.isJoined$.pipe(map((isJoined) => !isJoined)) + ? this.joined$.pipe(map((isJoined) => !isJoined)) : of(false); - private readonly data$: Observable = + private readonly data$ = this.scope.behavior( this.device.available$.pipe( map((available) => available.size > 0), distinctUntilChanged(), @@ -52,8 +53,8 @@ class MuteState { const set$ = new Subject(); const toggle$ = new Subject(); return { - set: (enabled: boolean) => set$.next(enabled), - toggle: () => toggle$.next(), + set: (enabled: boolean): void => set$.next(enabled), + toggle: (): void => toggle$.next(), // Assume the default value only once devices are actually connected enabled$: merge( set$, @@ -66,24 +67,24 @@ class MuteState { }; }, ), - this.scope.state(), - ); - - public readonly enabled$: Observable = this.data$.pipe( - switchMap(({ enabled$ }) => enabled$), + ), ); - public readonly setEnabled$: Observable<((enabled: boolean) => void) | null> = - this.data$.pipe(map(({ set }) => set)); + public readonly enabled$: Behavior = this.scope.behavior( + this.data$.pipe(switchMap(({ enabled$ }) => enabled$)), + ); - public readonly toggle$: Observable<(() => void) | null> = this.data$.pipe( - map(({ toggle }) => toggle), + public readonly setEnabled$: Behavior<((enabled: boolean) => void) | null> = + this.scope.behavior(this.data$.pipe(map(({ set }) => set))); + + public readonly toggle$: Behavior<(() => void) | null> = this.scope.behavior( + this.data$.pipe(map(({ toggle }) => toggle)), ); public constructor( private readonly scope: ObservableScope, - private readonly device: MediaDevice, - private readonly isJoined$: Observable, + private readonly device: MediaDevice, + private readonly joined$: Observable, private readonly enabledByConfig: boolean, ) {} } @@ -92,20 +93,20 @@ export class MuteStates { public readonly audio = new MuteState( this.scope, this.mediaDevices.audioInput, - this.isJoined$, + this.joined$, Config.get().media_devices.enable_video, ); public readonly video = new MuteState( this.scope, this.mediaDevices.videoInput, - this.isJoined$, + this.joined$, Config.get().media_devices.enable_video, ); public constructor( private readonly scope: ObservableScope, private readonly mediaDevices: MediaDevices, - private readonly isJoined$: Observable, + private readonly joined$: Observable, ) { if (widget !== null) { // Sync our mute states with the hosting client From e08f16f889ad9d6c2e8b427ae9ff86e2c4222af3 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 29 Aug 2025 18:46:24 +0200 Subject: [PATCH 020/144] All my Friday work. Demoable! --- src/room/InCallView.tsx | 35 ++-- src/room/LobbyView.tsx | 48 +++--- src/room/VideoPreview.tsx | 11 +- src/state/CallViewModel.ts | 81 ++++++---- src/state/Connection.ts | 243 +++++++--------------------- src/state/MuteStates.ts | 65 ++++++-- src/state/ObservableScope.ts | 7 + src/useCallViewKeyboardShortcuts.ts | 28 ++-- 8 files changed, 220 insertions(+), 298 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1c8b41e9..3d7ce8db 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -60,7 +60,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; -import { type MuteStates } from "./MuteStates"; +import { type MuteStates } from "../state/MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; @@ -143,6 +143,7 @@ export const ActiveCall: FC = (props) => { props.rtcSession, props.matrixRoom, mediaDevices, + props.muteStates, { encryptionSystem: props.e2eeSystem, autoLeaveWhenOthersLeft, @@ -161,6 +162,7 @@ export const ActiveCall: FC = (props) => { props.rtcSession, props.matrixRoom, mediaDevices, + props.muteStates, props.e2eeSystem, autoLeaveWhenOthersLeft, sendNotificationType, @@ -265,22 +267,19 @@ export const InCallView: FC = ({ ], ); - const toggleMicrophone = useCallback( - () => muteStates.audio.setEnabled?.((e) => !e), - [muteStates], - ); - const toggleCamera = useCallback( - () => muteStates.video.setEnabled?.((e) => !e), - [muteStates], - ); + const audioEnabled = useBehavior(muteStates.audio.enabled$); + const videoEnabled = useBehavior(muteStates.video.enabled$); + const toggleAudio = useBehavior(muteStates.audio.toggle$); + const toggleVideo = useBehavior(muteStates.video.toggle$); + const setAudioEnabled = useBehavior(muteStates.audio.setEnabled$); // This function incorrectly assumes that there is a camera and microphone, which is not always the case. // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! useCallViewKeyboardShortcuts( containerRef1, - toggleMicrophone, - toggleCamera, - (muted) => muteStates.audio.setEnabled?.(!muted), + toggleAudio, + toggleVideo, + setAudioEnabled, (reaction) => void sendReaction(reaction), () => void toggleRaisedHand(), ); @@ -764,18 +763,18 @@ export const InCallView: FC = ({ buttons.push( , , ); diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 391cb391..057c1206 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -31,7 +31,7 @@ import inCallStyles from "./InCallView.module.css"; import styles from "./LobbyView.module.css"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { type MatrixInfo, VideoPreview } from "./VideoPreview"; -import { type MuteStates } from "./MuteStates"; +import { type MuteStates } from "../state/MuteStates"; import { InviteButton } from "../button/InviteButton"; import { EndCallButton, @@ -50,8 +50,8 @@ import { useTrackProcessorSync, } from "../livekit/TrackProcessorContext"; import { usePageTitle } from "../usePageTitle"; -import { useLatest } from "../useLatest"; import { getValue } from "../utils/observable"; +import { useBehavior } from "../useBehavior"; interface Props { client: MatrixClient; @@ -88,14 +88,10 @@ export const LobbyView: FC = ({ const { t } = useTranslation(); usePageTitle(matrixInfo.roomName); - const onAudioPress = useCallback( - () => muteStates.audio.setEnabled?.((e) => !e), - [muteStates], - ); - const onVideoPress = useCallback( - () => muteStates.video.setEnabled?.((e) => !e), - [muteStates], - ); + const audioEnabled = useBehavior(muteStates.audio.enabled$); + const videoEnabled = useBehavior(muteStates.video.enabled$); + const toggleAudio = useBehavior(muteStates.audio.toggle$); + const toggleVideo = useBehavior(muteStates.video.toggle$); const [settingsModalOpen, setSettingsModalOpen] = useState(false); const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); @@ -133,7 +129,7 @@ export const LobbyView: FC = ({ // re-open the devices when they change (see below). const initialAudioOptions = useInitial( () => - muteStates.audio.enabled && { + audioEnabled && { deviceId: getValue(devices.audioInput.selected$)?.id, }, ); @@ -150,27 +146,21 @@ export const LobbyView: FC = ({ // We also pass in a clone because livekit mutates the object passed in, // which would cause the devices to be re-opened on the next render. audio: Object.assign({}, initialAudioOptions), - video: muteStates.video.enabled && { + video: videoEnabled && { deviceId: videoInputId, processor: initialProcessor, }, }), - [ - initialAudioOptions, - muteStates.video.enabled, - videoInputId, - initialProcessor, - ], + [initialAudioOptions, videoEnabled, videoInputId, initialProcessor], ); - const latestMuteStates = useLatest(muteStates); const onError = useCallback( (error: Error) => { logger.error("Error while creating preview Tracks:", error); - latestMuteStates.current.audio.setEnabled?.(false); - latestMuteStates.current.video.setEnabled?.(false); + muteStates.audio.setEnabled$.value?.(false); + muteStates.video.setEnabled$.value?.(false); }, - [latestMuteStates], + [muteStates], ); const tracks = usePreviewTracks(localTrackOptions, onError); @@ -217,7 +207,7 @@ export const LobbyView: FC = ({
diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 9d310864..ab083ffe 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -184,9 +184,6 @@ export const GroupCallView: FC = ({ } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting); - const [useExperimentalToDeviceTransport] = useSetting( - useExperimentalToDeviceTransportSetting, - ); // Save the password once we start the groupCallView useEffect(() => { @@ -223,12 +220,6 @@ export const GroupCallView: FC = ({ try { setJoined(true); // TODO-MULTI-SFU what to do with error handling now that we don't use this function? - // await enterRTCSession( - // rtcSession, - // perParticipantE2EE, - // useNewMembershipManager, - // useExperimentalToDeviceTransport, - // ); } catch (e) { if (e instanceof ElementCallError) { setExternalError(e); @@ -322,16 +313,16 @@ export const GroupCallView: FC = ({ const navigate = useNavigate(); - const onLeave = useCallback( - ( - cause: "user" | "error" = "user", - playSound: CallEventSounds = "left", - ): void => { + const onLeft = useCallback( + (reason: "timeout" | "user" | "allOthersLeft" | "decline"): void => { + let playSound: CallEventSounds = "left"; + if (reason === "timeout" || reason === "decline") playSound = reason; + + setLeft(true); const audioPromise = leaveSoundContext.current?.playSound(playSound); // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, // therefore we want the event to be sent instantly without getting queued/batched. const sendInstantly = !!widget; - setLeft(true); // we need to wait until the callEnded event is tracked on posthog. // Otherwise the iFrame gets killed before the callEnded event got tracked. const posthogRequest = new Promise((resolve) => { @@ -339,37 +330,33 @@ export const GroupCallView: FC = ({ room.roomId, rtcSession.memberships.length, sendInstantly, + rtcSession, ); window.setTimeout(resolve, 10); }); - // TODO-MULTI-SFU find a solution if this is supposed to happen here or in the view model. - leaveRTCSession( - rtcSession, - cause, - // Wait for the sound in widget mode (it's not long) - Promise.all([audioPromise, posthogRequest]), - ) - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - .then(async () => { + void Promise.all([audioPromise, posthogRequest]) + .then(() => { if ( !isPasswordlessUser && !confineToRoom && !PosthogAnalytics.instance.isEnabled() ) { - await navigate("/"); + void navigate("/"); } }) - .catch((e) => { - logger.error("Error leaving RTC session", e); - }); + .catch(() => + logger.error( + "could failed to play leave audio or send posthog leave event", + ), + ); }, [ leaveSoundContext, widget, - rtcSession, room.roomId, + rtcSession, isPasswordlessUser, confineToRoom, navigate, @@ -457,7 +444,7 @@ export const GroupCallView: FC = ({ matrixInfo={matrixInfo} rtcSession={rtcSession as MatrixRTCSession} matrixRoom={room} - onLeave={onLeave} + onLeft={onLeft} header={header} muteStates={muteStates} e2eeSystem={e2eeSystem} @@ -518,7 +505,8 @@ export const GroupCallView: FC = ({ }} onError={ (/**error*/) => { - if (rtcSession.isJoined()) onLeave("error"); + // TODO this should not be "user". It needs a new case + if (rtcSession.isJoined()) onLeft("user"); } } > diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 15a896a4..6d2aaf0a 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -177,7 +177,8 @@ function createInCallView(): RenderResult & { }} matrixRoom={room} livekitRoom={livekitRoom} - onLeave={function (): void { + participantCount={0} + onLeft={function (): void { throw new Error("Function not implemented."); }} onShareClick={null} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1e92d110..c12b44c9 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -23,11 +23,7 @@ import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; -import { - useObservable, - useObservableEagerState, - useSubscription, -} from "observable-hooks"; +import { useObservable, useObservableEagerState } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { @@ -94,10 +90,7 @@ import { } from "../reactions/useReactionsSender"; import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; import { ReactionsOverlay } from "./ReactionsOverlay"; -import { - CallEventAudioRenderer, - type CallEventSounds, -} from "./CallEventAudioRenderer"; +import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { debugTileLayout as debugTileLayoutSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, @@ -129,6 +122,8 @@ const maxTapDurationMs = 400; export interface ActiveCallProps extends Omit { e2eeSystem: EncryptionSystem; + // TODO refactor those reasons into an enum + onLeft: (reason: "user" | "timeout" | "decline" | "allOthersLeft") => void; } export const ActiveCall: FC = (props) => { @@ -154,8 +149,11 @@ export const ActiveCall: FC = (props) => { reactionsReader.reactions$, ); setVm(vm); + + const sub = vm.left$.subscribe(props.onLeft); return (): void => { vm.destroy(); + sub.unsubscribe(); reactionsReader.destroy(); }; }, [ @@ -167,6 +165,7 @@ export const ActiveCall: FC = (props) => { autoLeaveWhenOthersLeft, sendNotificationType, waitForCallPickup, + props.onLeft, ]); if (vm === null) return null; @@ -185,8 +184,6 @@ export interface InCallViewProps { rtcSession: MatrixRTCSession; matrixRoom: MatrixRoom; muteStates: MuteStates; - /** Function to call when the user explicitly ends the call */ - onLeave: (cause: "user", soundFile?: CallEventSounds) => void; header: HeaderStyle; otelGroupCallMembership?: OTelGroupCallMembership; onShareClick: (() => void) | null; @@ -199,7 +196,7 @@ export const InCallView: FC = ({ rtcSession, matrixRoom, muteStates, - onLeave, + header: headerStyle, onShareClick, }) => { @@ -295,7 +292,6 @@ export const InCallView: FC = ({ const showFooter = useBehavior(vm.showFooter$); const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); - useSubscription(vm.autoLeave$, () => onLeave("user")); // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; @@ -316,16 +312,6 @@ export const InCallView: FC = ({ }; }, [pickupPhaseAudio?.soundDuration, ringDuration]); - // When we enter timeout or decline we will leave the call. - useEffect((): void | (() => void) => { - if (callPickupState === "timeout") { - onLeave("user", "timeout"); - } - if (callPickupState === "decline") { - onLeave("user", "decline"); - } - }, [callPickupState, onLeave, pickupPhaseAudio]); - // When waiting for pickup, loop a waiting sound useEffect((): void | (() => void) => { if (callPickupState !== "ringing" || !pickupPhaseAudio) return; @@ -343,6 +329,7 @@ export const InCallView: FC = ({ if (callPickupState !== "ringing") return null; // Use room state for other participants data (the one that we likely want to reach) + // TODO: this screams it wants to be a behavior in the vm. const roomOthers = [ ...matrixRoom.getMembersWithMembership("join"), ...matrixRoom.getMembersWithMembership("invite"), @@ -816,7 +803,7 @@ export const InCallView: FC = ({ c.connectionState$), + startWith(ConnectionState.Disconnected), + ), + ); + // TODO-MULTI-SFU make sure that we consider the room memberships here as well (so that here we only have valid memberships) // this also makes it possible to use this memberships$ list in all observables based on it. // there should be no other call to: this.matrixRTCSession.memberships! @@ -541,12 +550,19 @@ export class CallViewModel extends ViewModel { this.join$.next(); } - private readonly leave$ = new Subject(); + private readonly leave$ = new Subject< + "decline" | "timeout" | "user" | "allOthersLeft" + >(); public leave(): void { - this.leave$.next(); + this.leave$.next("user"); } + private readonly _left$ = new Subject< + "decline" | "timeout" | "user" | "allOthersLeft" + >(); + public left$ = this._left$.asObservable(); + private readonly connectionInstructions$ = this.join$.pipe( switchMap(() => this.remoteConnections$), startWith(new Map()), @@ -628,10 +644,9 @@ export class CallViewModel extends ViewModel { private readonly connected$ = this.scope.behavior( and$( this.matrixConnected$, - // TODO-MULTI-SFU - // this.livekitConnectionState$.pipe( - // map((state) => state === ConnectionState.Connected), - // ), + this.livekitConnectionState$.pipe( + map((state) => state === ConnectionState.Connected), + ), ), ); @@ -663,7 +678,6 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - private readonly participants$ = this.scope.behavior< { participant: LocalParticipant | RemoteParticipant; @@ -731,7 +745,6 @@ export class CallViewModel extends ViewModel { // Handle room membership changes (and displayname updates) fromEvent(this.matrixRoom, RoomStateEvent.Members), // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ).pipe( startWith(null), map(() => { @@ -759,7 +772,7 @@ export class CallViewModel extends ViewModel { ); } return displaynameMap; - }, + }), ), ); @@ -971,21 +984,6 @@ export class CallViewModel extends ViewModel { this.memberships$.pipe(map((ms) => ms.length)), ); - private readonly allOthersLeft$ = this.memberships$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.every((m) => m.sender === this.userId) && - prev.some((m) => m.sender !== this.userId), - ), - map(() => {}), - take(1), - ); - - public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft - ? this.allOthersLeft$ - : NEVER; - private readonly didSendCallNotification$ = fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification, @@ -994,6 +992,7 @@ export class CallViewModel extends ViewModel { MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] > >; + /** * Whenever the RTC session tells us that it intends to ring the remote * participant's devices, this emits an Observable tracking the current state of @@ -1109,6 +1108,56 @@ export class CallViewModel extends ViewModel { map(() => {}), throttleTime(THROTTLE_SOUND_EFFECT_MS), ); + /** + * This observable tracks the matrix users that are currently in the call. + * There can be just one matrix user with multiple participants (see also participantChanges$) + */ + public readonly matrixUserChanges$ = this.userMedia$.pipe( + map( + (mediaItems) => + new Set( + mediaItems + .map((m) => m.vm.member?.userId) + .filter((id) => id !== undefined), + ), + ), + scan< + Set, + { + userIds: Set; + joinedUserIds: Set; + leftUserIds: Set; + } + >( + (prevState, userIds) => { + const left = new Set( + [...prevState.userIds].filter((id) => !userIds.has(id)), + ); + const joined = new Set( + [...userIds].filter((id) => !prevState.userIds.has(id)), + ); + return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; + }, + { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, + ), + ); + + private readonly allOthersLeft$ = this.matrixUserChanges$.pipe( + map(({ userIds, leftUserIds }) => { + if (!this.userId) { + logger.warn("Could not access user ID to compute allOthersLeft"); + return false; + } + return ( + userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 + ); + }), + startWith(false), + ); + + public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft + ? this.allOthersLeft$ + : NEVER; /** * List of MediaItems that we want to display, that are of type ScreenShare @@ -1791,8 +1840,6 @@ export class CallViewModel extends ViewModel { ), filter((v) => v.playSounds), ); - // TODO-REBASE: expose connection state observable - public readonly livekitConnectionState$: Observable; public constructor( // A call is permanently tied to a single Matrix room @@ -1839,18 +1886,34 @@ export class CallViewModel extends ViewModel { ); }); - this.join$.pipe(this.scope.bind()).subscribe(() => { - // TODO-MULTI-SFU: this makes no sense what so ever!!! - // need to look into this again. - // leaveRTCSession( - // this.matrixRTCSession, - // "user", // TODO-MULTI-SFU ? - // // Wait for the sound in widget mode (it's not long) - // Promise.resolve(), // TODO-MULTI-SFU - // //Promise.all([audioPromise, posthogRequest]), - // ).catch((e) => { - // logger.error("Error leaving RTC session", e); - // }); + this.allOthersLeft$ + .pipe( + this.scope.bind(), + filter((l) => (l && this.options.autoLeaveWhenOthersLeft) ?? false), + distinctUntilChanged(), + ) + .subscribe(() => { + this.leave$.next("allOthersLeft"); + }); + + this.callPickupState$.pipe(this.scope.bind()).subscribe((state) => { + if (state === "timeout" || state === "decline") { + this.leave$.next(state); + } + }); + + this.leave$.pipe(this.scope.bind()).subscribe((reason) => { + const { confineToRoom } = getUrlParams(); + leaveRTCSession(this.matrixRTCSession, "user") + // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. + .then(() => { + if (!confineToRoom && !PosthogAnalytics.instance.isEnabled()) { + this._left$.next(reason); + } + }) + .catch((e) => { + logger.error("Error leaving RTC session", e); + }); }); // Pause upstream of all local media tracks when we're disconnected from From 41e152f4207d28be530feb9e895c134ac5a9f814 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 17 Sep 2025 11:25:49 +0200 Subject: [PATCH 025/144] dont throw disconnected error at start of the call Signed-off-by: Timo K --- src/room/InCallView.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index c12b44c9..2f720148 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -209,8 +209,9 @@ export const InCallView: FC = ({ // annoyingly we don't get the disconnection reason this way, // only by listening for the emitted event - if (connectionState === ConnectionState.Disconnected) - throw new ConnectionLostError(); + // This needs to be done differential. with the vm connection state we start with Disconnected. + // if (connectionState === ConnectionState.Disconnected) + // throw new ConnectionLostError(); const containerRef1 = useRef(null); const [containerRef2, bounds] = useMeasure(); From d9fe31039ff1bdbb2ffb4ea470e7d9289fe71afe Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 19 Sep 2025 18:01:45 +0200 Subject: [PATCH 026/144] start fixing CallViewModel tests. Signed-off-by: Timo K --- src/room/InCallView.tsx | 1 + src/room/VideoPreview.test.tsx | 13 +++---------- src/state/CallViewModel.test.ts | 12 +++--------- src/utils/test.ts | 8 ++++++++ 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 2f720148..daf5034a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -210,6 +210,7 @@ export const InCallView: FC = ({ // annoyingly we don't get the disconnection reason this way, // only by listening for the emitted event // This needs to be done differential. with the vm connection state we start with Disconnected. + // TODO-MULTI-SFU decide how to handle this properly // if (connectionState === ConnectionState.Disconnected) // throw new ConnectionLostError(); diff --git a/src/room/VideoPreview.test.tsx b/src/room/VideoPreview.test.tsx index 3bbb6ad5..717333ee 100644 --- a/src/room/VideoPreview.test.tsx +++ b/src/room/VideoPreview.test.tsx @@ -9,15 +9,8 @@ import { expect, describe, it, vi, beforeAll } from "vitest"; import { render } from "@testing-library/react"; import { type MatrixInfo, VideoPreview } from "./VideoPreview"; -import { type MuteStates } from "./MuteStates"; import { E2eeType } from "../e2ee/e2eeType"; - -function mockMuteStates({ audio = true, video = true } = {}): MuteStates { - return { - audio: { enabled: audio, setEnabled: vi.fn() }, - video: { enabled: video, setEnabled: vi.fn() }, - }; -} +import { mockMuteStates } from "../utils/test"; describe("VideoPreview", () => { const matrixInfo: MatrixInfo = { @@ -49,7 +42,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, @@ -61,7 +54,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index ef4ef762..b736b780 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -53,7 +53,6 @@ import { type Layout, } from "./CallViewModel"; import { - mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, mockMatrixRoomMember, @@ -62,6 +61,7 @@ import { mockRtcMembership, MockRTCSession, mockMediaDevices, + mockMuteStates, } from "../utils/test"; import { ECAddonConnectionState, @@ -340,21 +340,15 @@ function withCallViewModel( const roomEventSelectorSpy = vi .spyOn(ComponentsCore, "roomEventSelector") .mockImplementation((_room, _eventType) => of()); - - const livekitRoom = mockLivekitRoom( - { localParticipant }, - { remoteParticipants$ }, - ); - + const muteStates = mockMuteStates(); const raisedHands$ = new BehaviorSubject>({}); const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, room, - livekitRoom, mediaDevices, + muteStates, options, - connectionState$, raisedHands$, new BehaviorSubject({}), ); diff --git a/src/utils/test.ts b/src/utils/test.ts index 31c6068a..53b6d0ee 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -53,6 +53,7 @@ import { Config } from "../config/Config"; import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; +import { Handler, MuteStates } from "../state/MuteStates"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -417,3 +418,10 @@ export function mockMediaDevices(data: Partial): MediaDevices { ...data, } as MediaDevices; } + +export function mockMuteStates( + joined$: Observable = of(true), +): MuteStates { + const observableScope = new ObservableScope(); + return new MuteStates(observableScope, mockMediaDevices({}), joined$); +} From 02f23e25fd868bd1329667da3c714944b614f69f Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Sep 2025 14:16:24 +0200 Subject: [PATCH 027/144] remove todo from matrix audio renderer Signed-off-by: Timo K --- src/livekit/MatrixAudioRenderer.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 8c4c9e10..0b87b27b 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -81,8 +81,6 @@ export function LivekitRoomAudioRenderer({ loggedInvalidIdentities.current.add(identity); }; - // TODO-MULTI-SFU this uses the livekit room form the context. We need to change it so it uses the - // livekit room explicitly so we can pass a list of rooms into the audio renderer and call useTracks for each room. const tracks = useTracks( [ Track.Source.Microphone, From dddda7057f68922d663e92e7c2614b2767a7dd09 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Sep 2025 14:17:38 +0200 Subject: [PATCH 028/144] add todo comments and who works on them Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 1 + src/room/InCallView.tsx | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index ab083ffe..367f72a1 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -220,6 +220,7 @@ export const GroupCallView: FC = ({ try { setJoined(true); // TODO-MULTI-SFU what to do with error handling now that we don't use this function? + // @BillCarsonFr } catch (e) { if (e instanceof ElementCallError) { setExternalError(e); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index daf5034a..4daeae83 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -211,6 +211,7 @@ export const InCallView: FC = ({ // only by listening for the emitted event // This needs to be done differential. with the vm connection state we start with Disconnected. // TODO-MULTI-SFU decide how to handle this properly + // @BillCarsonFr // if (connectionState === ConnectionState.Disconnected) // throw new ConnectionLostError(); @@ -738,6 +739,7 @@ export const InCallView: FC = ({ const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const memberships = useBehavior(vm.memberships$); const toggleScreensharing = useCallback(() => { + // TODO-MULTI-SFU implement screensharing throw new Error("TODO-MULTI-SFU"); // localParticipant // .setScreenShareEnabled(!isScreenShareEnabled, { From 8bf24895ce42554568eef258cb21af455c61e099 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Sep 2025 14:18:23 +0200 Subject: [PATCH 029/144] TODO: settings modal with multiple connections Signed-off-by: Timo K --- src/room/InCallView.tsx | 2 +- src/settings/DeveloperSettingsTab.tsx | 33 +++++++++++++++------------ src/settings/SettingsModal.tsx | 12 +++++++--- src/state/CallViewModel.ts | 9 +++++++- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 4daeae83..b8460ad8 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -897,7 +897,7 @@ export const InCallView: FC = ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - livekitRoom={undefined} // TODO-MULTI-SFU + livekitRooms={allLivekitRooms} /> )} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 36df5c39..d503385b 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -26,10 +26,10 @@ import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; interface Props { client: MatrixClient; - livekitRoom?: LivekitRoom; + livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[]; } -export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => { +export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { const { t } = useTranslation(); const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); const [debugTileLayout, setDebugTileLayout] = useSetting( @@ -59,15 +59,16 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => { const urlParams = useUrlParams(); - const sfuUrl = useMemo((): URL | null => { - if (livekitRoom?.engine.client.ws?.url) { + const localSfuUrl = useMemo((): URL | null => { + const localRoom = livekitRooms?.find((r) => r.isLocal)?.room; + if (localRoom?.engine.client.ws?.url) { // strip the URL params - const url = new URL(livekitRoom.engine.client.ws.url); + const url = new URL(localRoom.engine.client.ws.url); url.search = ""; return url; } return null; - }, [livekitRoom]); + }, [livekitRooms]); return ( <> @@ -211,22 +212,26 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => { )} />{" "} - {livekitRoom ? ( + {livekitRooms?.map((livekitRoom) => ( <> -

+

{t("developer_mode.livekit_sfu", { - url: sfuUrl?.href || "unknown", + url: livekitRoom.url || "unknown", })} +

+ {livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

} +

+ {t("developer_mode.livekit_server_info")}( + {livekitRoom.isLocal ? "local" : "remote"})

-

{t("developer_mode.livekit_server_info")}

-            {livekitRoom.serverInfo
-              ? JSON.stringify(livekitRoom.serverInfo, null, 2)
+            {livekitRoom.room.serverInfo
+              ? JSON.stringify(livekitRoom.room.serverInfo, null, 2)
               : "undefined"}
-            {livekitRoom.metadata}
+            {livekitRoom.room.metadata}
- ) : null} + ))}

{t("developer_mode.environment_variables")}

{JSON.stringify(import.meta.env, null, 2)}

{t("developer_mode.url_params")}

diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 3272200d..9e581647 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -51,7 +51,11 @@ interface Props { onTabChange: (tab: SettingsTab) => void; client: MatrixClient; roomId?: string; - livekitRoom?: LivekitRoom; + livekitRooms?: { + room: LivekitRoom; + url: string; + isLocal?: boolean; + }[]; } export const defaultSettingsTab: SettingsTab = "audio"; @@ -63,7 +67,7 @@ export const SettingsModal: FC = ({ onTabChange, client, roomId, - livekitRoom, + livekitRooms, }) => { const { t } = useTranslation(); @@ -204,7 +208,9 @@ export const SettingsModal: FC = ({ const developerTab: Tab = { key: "developer", name: t("settings.developer_tab_title"), - content: , + content: ( + + ), }; const tabs = [audioTab, videoTab]; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6e922bee..8479f76b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -593,11 +593,18 @@ export class CallViewModel extends ViewModel { ]).pipe( map(([remoteConnections, localConnection, localFocus]) => Array.from(remoteConnections.entries()) - .map(([index, c]) => ({ room: c.livekitRoom, url: index })) + .map( + ([index, c]) => + ({ + room: c.livekitRoom, + url: index, + }) as { room: LivekitRoom; url: string; isLocal?: boolean }, + ) .concat([ { room: localConnection.livekitRoom, url: localFocus.livekit_service_url, + isLocal: true, }, ]), ), From 78e9521f22f6c428b009ff98d256be1b56ef4694 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 11:38:34 +0200 Subject: [PATCH 030/144] Make track processor work Signed-off-by: Timo K --- src/livekit/TrackProcessorContext.tsx | 42 ++++++++++++++++++++++++++- src/livekit/useLivekit.ts | 1 + src/room/InCallView.tsx | 4 +++ src/state/CallViewModel.ts | 3 ++ src/state/Connection.ts | 21 ++++++++++++-- 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index b37a6e3e..4a5ace46 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -19,14 +19,21 @@ import { useMemo, } from "react"; import { type LocalVideoTrack } from "livekit-client"; +import { combineLatest, map, type Observable } from "rxjs"; +import { useObservable } from "observable-hooks"; import { backgroundBlur as backgroundBlurSettings, useSetting, } from "../settings/settings"; import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; +import { type Behavior } from "../state/Behavior"; -type ProcessorState = { +//TODO-MULTI-SFU: This is not yet fully there. +// it is a combination of exposing observable and react hooks. +// preferably we should not make this a context anymore and instead just a vm? + +export type ProcessorState = { supported: boolean | undefined; processor: undefined | ProcessorWrapper; }; @@ -42,6 +49,39 @@ export function useTrackProcessor(): ProcessorState { return state; } +export function useTrackProcessorObservable$(): Observable { + const state = use(ProcessorContext); + if (state === undefined) + throw new Error( + "useTrackProcessor must be used within a ProcessorProvider", + ); + const state$ = useObservable( + (init$) => init$.pipe(map(([init]) => init)), + [state], + ); + + return state$; +} + +export const trackProcessorSync = ( + videoTrack$: Behavior, + processor$: Behavior, +): void => { + combineLatest([videoTrack$, processor$]).subscribe( + ([videoTrack, processorState]) => { + if (!processorState) return; + if (!videoTrack) return; + const { processor } = processorState; + if (processor && !videoTrack.getProcessor()) { + void videoTrack.setProcessor(processor); + } + if (!processor && videoTrack.getProcessor()) { + void videoTrack.stopProcessor(); + } + }, + ); +}; + export const useTrackProcessorSync = ( videoTrack: LocalVideoTrack | null, ): void => { diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts index 0672a8eb..420bac95 100644 --- a/src/livekit/useLivekit.ts +++ b/src/livekit/useLivekit.ts @@ -55,6 +55,7 @@ interface UseLivekitResult { } // TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object. +// seems to be mostly done... See Connection.ts export function useLivekitPublicationRoom( rtcSession: MatrixRTCSession, muteStates: MuteStates, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b8460ad8..157ee46a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -114,6 +114,7 @@ import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; import { ConnectionLostError } from "../utils/errors.ts"; +import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -133,6 +134,7 @@ export const ActiveCall: FC = (props) => { const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = useUrlParams(); + const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { const reactionsReader = new ReactionsReader(props.rtcSession); const vm = new CallViewModel( @@ -147,6 +149,7 @@ export const ActiveCall: FC = (props) => { }, reactionsReader.raisedHands$, reactionsReader.reactions$, + trackProcessorState$, ); setVm(vm); @@ -166,6 +169,7 @@ export const ActiveCall: FC = (props) => { sendNotificationType, waitForCallPickup, props.onLeft, + trackProcessorState$, ]); if (vm === null) return null; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8479f76b..40828357 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -125,6 +125,7 @@ import { Connection, PublishConnection } from "./Connection"; import { type MuteStates } from "./MuteStates"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { getUrlParams } from "../UrlParams"; +import { type ProcessorState } from "../livekit/TrackProcessorContext"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -460,6 +461,7 @@ export class CallViewModel extends ViewModel { this.mediaDevices, this.muteStates, this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), ), ); @@ -1861,6 +1863,7 @@ export class CallViewModel extends ViewModel { private readonly reactionsSubject$: Observable< Record >, + private readonly trackProcessorState$: Observable, ) { super(); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 4f9721ea..6804b2b7 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -15,6 +15,7 @@ import { Room as LivekitRoom, type E2EEOptions, Track, + LocalVideoTrack, } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { @@ -39,6 +40,11 @@ import { defaultLiveKitOptions } from "../livekit/options"; import { getValue } from "../utils/observable"; import { getUrlParams } from "../UrlParams"; import { type MuteStates } from "./MuteStates"; +import { + type ProcessorState, + trackProcessorSync, +} from "../livekit/TrackProcessorContext"; +import { observeTrackReference$ } from "./MediaViewModel"; export class Connection { protected stopped = false; @@ -151,6 +157,7 @@ export class PublishConnection extends Connection { devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, + trackerProcessorState$: Behavior, ) { logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); @@ -160,8 +167,7 @@ export class PublishConnection extends Connection { videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, deviceId: devices.videoInput.selected$.value?.id, - // TODO-MULTI-SFU add processor support back - // processor, + processor: trackerProcessorState$.value.processor, }, audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, @@ -191,6 +197,17 @@ export class PublishConnection extends Connection { room, ); + // Setup track processor syncing (blur) + const track$ = this.scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( + map((trackRef) => { + const track = trackRef?.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }), + ), + ); + trackProcessorSync(track$, trackerProcessorState$); + this.muteStates.audio.setHandler(async (desired) => { try { await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); From 7777179935f83cc0c2e79663c7a3ed3036b89a98 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 11:40:29 +0200 Subject: [PATCH 031/144] cleanup (delete files useLivekit) now covered by Connection.ts Signed-off-by: Timo K --- src/livekit/livekitSubscriptionRoom.ts | 123 ------- src/livekit/useLivekit.ts | 441 ------------------------- src/state/Connection.ts | 2 - 3 files changed, 566 deletions(-) delete mode 100644 src/livekit/livekitSubscriptionRoom.ts delete mode 100644 src/livekit/useLivekit.ts diff --git a/src/livekit/livekitSubscriptionRoom.ts b/src/livekit/livekitSubscriptionRoom.ts deleted file mode 100644 index f92ff10e..00000000 --- a/src/livekit/livekitSubscriptionRoom.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ConnectionState, - type E2EEManagerOptions, - ExternalE2EEKeyProvider, - LocalVideoTrack, - Room, - type RoomOptions, -} from "livekit-client"; -import { useEffect, useRef } from "react"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; - -import { defaultLiveKitOptions } from "./options"; -import { type SFUConfig } from "./openIDSFU"; -import { type MuteStates } from "../room/MuteStates"; -import { useMediaDevices } from "../MediaDevicesContext"; -import { - type ECConnectionState, - useECConnectionState, -} from "./useECConnectionState"; -import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { E2eeType } from "../e2ee/e2eeType"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { - useTrackProcessor, - useTrackProcessorSync, -} from "./TrackProcessorContext"; -import { observeTrackReference$ } from "../state/MediaViewModel"; -import { useUrlParams } from "../UrlParams"; -import { useInitial } from "../useInitial"; -import { getValue } from "../utils/observable"; -import { type SelectedDevice } from "../state/MediaDevices"; - -interface UseLivekitResult { - livekitPublicationRoom?: Room; - connState: ECConnectionState; -} - -// TODO-MULTI-SFU This is all the logic we need in the subscription connection logic (sync output devices) -// This is not used! (but summarizes what we need) -export function livekitSubscriptionRoom( - rtcSession: MatrixRTCSession, - muteStates: MuteStates, - sfuConfig: SFUConfig | undefined, - e2eeSystem: EncryptionSystem, -): UseLivekitResult { - // Only ever create the room once via useInitial. - // The call can end up with multiple livekit rooms. This is the particular room in - // which this participant publishes their media. - const publicationRoom = useInitial(() => { - logger.info("[LivekitRoom] Create LiveKit room"); - - let e2ee: E2EEManagerOptions | undefined; - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - logger.info("Created MatrixKeyProvider (per participant)"); - e2ee = { - keyProvider: new MatrixKeyProvider(), - worker: new E2EEWorker(), - }; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - logger.info("Created ExternalE2EEKeyProvider (shared key)"); - e2ee = { - keyProvider: new ExternalE2EEKeyProvider(), - worker: new E2EEWorker(), - }; - } - - const roomOptions: RoomOptions = { - ...defaultLiveKitOptions, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee, - }; - // We have to create the room manually here due to a bug inside - // @livekit/components-react. JSON.stringify() is used in deps of a - // useEffect() with an argument that references itself, if E2EE is enabled - const room = new Room(roomOptions); - room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - return room; - }); - - // Setup and update the keyProvider which was create by `createRoom` - useEffect(() => { - const e2eeOptions = publicationRoom.options.e2ee; - if ( - e2eeSystem.kind === E2eeType.NONE || - !(e2eeOptions && "keyProvider" in e2eeOptions) - ) - return; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession); - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - (e2eeOptions.keyProvider as ExternalE2EEKeyProvider) - .setKey(e2eeSystem.secret) - .catch((e) => { - logger.error("Failed to set shared key for E2EE", e); - }); - } - }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); - - return { - connState: connectionState, - livekitPublicationRoom: publicationRoom, - }; -} diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts deleted file mode 100644 index 420bac95..00000000 --- a/src/livekit/useLivekit.ts +++ /dev/null @@ -1,441 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ConnectionState, - type E2EEManagerOptions, - ExternalE2EEKeyProvider, - type LocalTrackPublication, - LocalVideoTrack, - Room, - type RoomOptions, - Track, -} from "livekit-client"; -import { useEffect, useRef } from "react"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { useObservable, useObservableEagerState } from "observable-hooks"; -import { - map, - NEVER, - type Observable, - type Subscription, - switchMap, -} from "rxjs"; - -import { defaultLiveKitOptions } from "./options"; -import { type SFUConfig } from "./openIDSFU"; -import { type MuteStates } from "../room/MuteStates"; -import { useMediaDevices } from "../MediaDevicesContext"; -import { - type ECConnectionState, - useECConnectionState, -} from "./useECConnectionState"; -import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { E2eeType } from "../e2ee/e2eeType"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { - useTrackProcessor, - useTrackProcessorSync, -} from "./TrackProcessorContext"; -import { observeTrackReference$ } from "../state/MediaViewModel"; -import { useUrlParams } from "../UrlParams"; -import { useInitial } from "../useInitial"; -import { getValue } from "../utils/observable"; -import { type SelectedDevice } from "../state/MediaDevices"; - -interface UseLivekitResult { - livekitPublicationRoom?: Room; - connState: ECConnectionState; -} - -// TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object. -// seems to be mostly done... See Connection.ts -export function useLivekitPublicationRoom( - rtcSession: MatrixRTCSession, - muteStates: MuteStates, - sfuConfig: SFUConfig | undefined, - e2eeSystem: EncryptionSystem, -): UseLivekitResult { - const { controlledAudioDevices } = useUrlParams(); - - const initialMuteStates = useInitial(() => muteStates); - - const devices = useMediaDevices(); - const initialAudioInputId = useInitial( - () => getValue(devices.audioInput.selected$)?.id, - ); - - // Store if audio/video are currently updating. If to prohibit unnecessary calls - // to setMicrophoneEnabled/setCameraEnabled - const audioMuteUpdating = useRef(false); - const videoMuteUpdating = useRef(false); - // Store the current button mute state that gets passed to this hook via props. - // We need to store it for awaited code that relies on the current value. - const buttonEnabled = useRef({ - audio: initialMuteStates.audio.enabled, - video: initialMuteStates.video.enabled, - }); - - const { processor } = useTrackProcessor(); - - // Only ever create the room once via useInitial. - // The call can end up with multiple livekit rooms. This is the particular room in - // which this participant publishes their media. - const publicationRoom = useInitial(() => { - logger.info("[LivekitRoom] Create LiveKit room"); - - let e2ee: E2EEManagerOptions | undefined; - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - logger.info("Created MatrixKeyProvider (per participant)"); - e2ee = { - keyProvider: new MatrixKeyProvider(), - worker: new E2EEWorker(), - }; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - logger.info("Created ExternalE2EEKeyProvider (shared key)"); - e2ee = { - keyProvider: new ExternalE2EEKeyProvider(), - worker: new E2EEWorker(), - }; - } - - const roomOptions: RoomOptions = { - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: getValue(devices.videoInput.selected$)?.id, - processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: initialAudioInputId, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee, - }; - // We have to create the room manually here due to a bug inside - // @livekit/components-react. JSON.stringify() is used in deps of a - // useEffect() with an argument that references itself, if E2EE is enabled - const room = new Room(roomOptions); - room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - return room; - }); - - // Setup and update the keyProvider which was create by `createRoom` - useEffect(() => { - const e2eeOptions = publicationRoom.options.e2ee; - if ( - e2eeSystem.kind === E2eeType.NONE || - !(e2eeOptions && "keyProvider" in e2eeOptions) - ) - return; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession); - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - (e2eeOptions.keyProvider as ExternalE2EEKeyProvider) - .setKey(e2eeSystem.secret) - .catch((e) => { - logger.error("Failed to set shared key for E2EE", e); - }); - } - }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); - - // Sync the requested track processors with LiveKit - useTrackProcessorSync( - useObservableEagerState( - useObservable( - (room$) => - room$.pipe( - switchMap(([room]) => - observeTrackReference$( - room.localParticipant, - Track.Source.Camera, - ), - ), - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ), - [publicationRoom], - ), - ), - ); - - const connectionState = useECConnectionState( - initialAudioInputId, - initialMuteStates.audio.enabled, - publicationRoom, - sfuConfig, - ); - - // Log errors when local participant has issues publishing a track. - useEffect(() => { - const localTrackUnpublishedFn = ( - publication: LocalTrackPublication, - ): void => { - logger.info( - "Local track unpublished", - publication.trackName, - publication.trackInfo, - ); - }; - const mediaDevicesErrorFn = (error: Error): void => { - logger.warn("Media devices error when publishing a track", error); - }; - - room.localParticipant.on("localTrackUnpublished", localTrackUnpublishedFn); - room.localParticipant.on("mediaDevicesError", mediaDevicesErrorFn); - - return (): void => { - room.localParticipant.off( - "localTrackUnpublished", - localTrackUnpublishedFn, - ); - room.localParticipant.off("mediaDevicesError", mediaDevicesErrorFn); - }; - }, [room.localParticipant]); - - useEffect(() => { - // Sync the requested mute states with LiveKit's mute states. We do it this - // way around rather than using LiveKit as the source of truth, so that the - // states can be consistent throughout the lobby and loading screens. - // It's important that we only do this in the connected state, because - // LiveKit's internal mute states aren't consistent during connection setup, - // and setting tracks to be enabled during this time causes errors. - if ( - publicationRoom !== undefined && - connectionState === ConnectionState.Connected - ) { - const participant = publicationRoom.localParticipant; - // Always update the muteButtonState Ref so that we can read the current - // state in awaited blocks. - buttonEnabled.current = { - audio: muteStates.audio.enabled, - video: muteStates.video.enabled, - }; - - enum MuteDevice { - Microphone, - Camera, - } - - const syncMuteState = async ( - iterCount: number, - type: MuteDevice, - ): Promise => { - // The approach for muting is to always bring the actual livekit state in sync with the button - // This allows for a very predictable and reactive behavior for the user. - // (the new state is the old state when pressing the button n times (where n is even)) - // (the new state is different to the old state when pressing the button n times (where n is uneven)) - // In case there are issues with the device there might be situations where setMicrophoneEnabled/setCameraEnabled - // return immediately. This should be caught with the Error("track with new mute state could not be published"). - // For now we are still using an iterCount to limit the recursion loop to 10. - // This could happen if the device just really does not want to turn on (hardware based issue) - // but the mute button is in unmute state. - // For now our fail mode is to just stay in this state. - // TODO: decide for a UX on how that fail mode should be treated (disable button, hide button, sync button back to muted without user input) - - if (iterCount > 10) { - logger.error( - "Stop trying to sync the input device with current mute state after 10 failed tries", - ); - return; - } - let devEnabled; - let btnEnabled; - let updating; - switch (type) { - case MuteDevice.Microphone: - devEnabled = participant.isMicrophoneEnabled; - btnEnabled = buttonEnabled.current.audio; - updating = audioMuteUpdating.current; - break; - case MuteDevice.Camera: - devEnabled = participant.isCameraEnabled; - btnEnabled = buttonEnabled.current.video; - updating = videoMuteUpdating.current; - break; - } - if (devEnabled !== btnEnabled && !updating) { - try { - let trackPublication; - switch (type) { - case MuteDevice.Microphone: - audioMuteUpdating.current = true; - trackPublication = await participant.setMicrophoneEnabled( - buttonEnabled.current.audio, - publicationRoom.options.audioCaptureDefaults, - ); - audioMuteUpdating.current = false; - break; - case MuteDevice.Camera: - videoMuteUpdating.current = true; - trackPublication = await participant.setCameraEnabled( - buttonEnabled.current.video, - publicationRoom.options.videoCaptureDefaults, - ); - videoMuteUpdating.current = false; - break; - } - - if (trackPublication) { - // await participant.setMicrophoneEnabled can return immediately in some instances, - // so that participant.isMicrophoneEnabled !== buttonEnabled.current.audio still holds true. - // This happens if the device is still in a pending state - // "sleeping" here makes sure we let react do its thing so that participant.isMicrophoneEnabled is updated, - // so we do not end up in a recursion loop. - await new Promise((r) => setTimeout(r, 100)); - - // track got successfully changed to mute/unmute - // Run the check again after the change is done. Because the user - // can update the state (presses mute button) while the device is enabling - // itself we need might need to update the mute state right away. - // This async recursion makes sure that setCamera/MicrophoneEnabled is - // called as little times as possible. - await syncMuteState(iterCount + 1, type); - } else { - throw new Error( - "track with new mute state could not be published", - ); - } - } catch (e) { - if ((e as DOMException).name === "NotAllowedError") { - logger.error( - "Fatal error while syncing mute state: resetting", - e, - ); - if (type === MuteDevice.Microphone) { - audioMuteUpdating.current = false; - muteStates.audio.setEnabled?.(false); - } else { - videoMuteUpdating.current = false; - muteStates.video.setEnabled?.(false); - } - } else { - logger.error( - "Failed to sync audio mute state with LiveKit (will retry to sync in 1s):", - e, - ); - setTimeout(() => { - syncMuteState(iterCount + 1, type).catch((e) => { - logger.error( - `Failed to sync ${MuteDevice[type]} mute state with LiveKit iterCount=${iterCount + 1}`, - e, - ); - }); - }, 1000); - } - } - } - }; - - syncMuteState(0, MuteDevice.Microphone).catch((e) => { - logger.error("Failed to sync audio mute state with LiveKit", e); - }); - syncMuteState(0, MuteDevice.Camera).catch((e) => { - logger.error("Failed to sync video mute state with LiveKit", e); - }); - } - }, [publicationRoom, muteStates, connectionState]); - - useEffect(() => { - // Sync the requested devices with LiveKit's devices - if ( - publicationRoom !== undefined && - connectionState === ConnectionState.Connected - ) { - const syncDevice = ( - kind: MediaDeviceKind, - selected$: Observable, - ): Subscription => - selected$.subscribe((device) => { - logger.info( - "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - publicationRoom.getActiveDevice(kind), - " !== ", - device?.id, - ); - if ( - device !== undefined && - publicationRoom.getActiveDevice(kind) !== device.id - ) { - publicationRoom - .switchActiveDevice(kind, device.id) - .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e), - ); - } - }); - - const subscriptions = [ - syncDevice("audioinput", devices.audioInput.selected$), - !controlledAudioDevices - ? syncDevice("audiooutput", devices.audioOutput.selected$) - : undefined, - syncDevice("videoinput", devices.videoInput.selected$), - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. - devices.audioInput.selected$ - .pipe(switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER)) - .subscribe(() => { - const activeMicTrack = Array.from( - publicationRoom.localParticipant.audioTrackPublications.values(), - ).find((d) => d.source === Track.Source.Microphone)?.track; - - if ( - activeMicTrack && - // only restart if the stream is still running: LiveKit will detect - // when a track stops & restart appropriately, so this is not our job. - // Plus, we need to avoid restarting again if the track is already in - // the process of being restarted. - activeMicTrack.mediaStreamTrack.readyState !== "ended" - ) { - // Restart the track, which will cause Livekit to do another - // getUserMedia() call with deviceId: default to get the *new* default device. - // Note that room.switchActiveDevice() won't work: Livekit will ignore it because - // the deviceId hasn't changed (was & still is default). - publicationRoom.localParticipant - .getTrackPublication(Track.Source.Microphone) - ?.audioTrack?.restartTrack() - .catch((e) => { - logger.error(`Failed to restart audio device track`, e); - }); - } - }), - ]; - - return (): void => { - for (const s of subscriptions) s?.unsubscribe(); - }; - } - }, [publicationRoom, devices, connectionState, controlledAudioDevices]); - - return { - connState: connectionState, - livekitPublicationRoom: publicationRoom, - }; -} diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 6804b2b7..e14ddc9a 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -1,4 +1,3 @@ -// TODO-MULTI-SFU Add all device syncing logic from useLivekit /* Copyright 2025 New Vector Ltd. @@ -294,5 +293,4 @@ export class PublishConnection extends Connection { } }); } - // TODO-MULTI-SFU Sync the requested track processors with LiveKit } From 96e96a5e43b9d6a6b8b2b0a3428a5fe6dc82eeab Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 12:25:05 +0200 Subject: [PATCH 032/144] fix leaving Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 367f72a1..614c7b50 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -68,7 +68,6 @@ import { import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx"; import { useNewMembershipManager as useNewMembershipManagerSetting, - useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, useSetting, } from "../settings/settings"; import { useTypedEventEmitter } from "../useEvents"; @@ -310,6 +309,7 @@ export const GroupCallView: FC = ({ useNewMembershipManager, ]); + // TODO refactor this + "joined" to just one callState const [left, setLeft] = useState(false); const navigate = useNavigate(); @@ -319,6 +319,7 @@ export const GroupCallView: FC = ({ let playSound: CallEventSounds = "left"; if (reason === "timeout" || reason === "decline") playSound = reason; + setJoined(false); setLeft(true); const audioPromise = leaveSoundContext.current?.playSound(playSound); // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, @@ -354,6 +355,7 @@ export const GroupCallView: FC = ({ ); }, [ + setJoined, leaveSoundContext, widget, room.roomId, From 6b44f3b0081ad473603cfeb82d04c8b276bd41ae Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 12:25:31 +0200 Subject: [PATCH 033/144] a tiny bit of tests lint fixes. Signed-off-by: Timo K --- src/utils/test-viewmodel.ts | 22 ++++++---------------- src/utils/test.ts | 2 +- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 687adba7..785cbe1b 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState } from "livekit-client"; import { type CallMembership, type MatrixRTCSession, @@ -27,19 +26,13 @@ import { type CallViewModelOptions, } from "../state/CallViewModel"; import { - mockLivekitRoom, mockMatrixRoom, mockMediaDevices, + mockMuteStates, MockRTCSession, } from "./test"; -import { - aliceRtcMember, - aliceParticipant, - localParticipant, - localRtcMember, -} from "./test-fixtures"; +import { aliceRtcMember, localRtcMember } from "./test-fixtures"; import { type RaisedHandInfo, type ReactionInfo } from "../reactions"; -import { constant } from "../state/Behavior"; export function getBasicRTCSession( members: RoomMember[], @@ -141,23 +134,20 @@ export function getBasicCallViewModelEnvironment( const handRaisedSubject$ = new BehaviorSubject({}); const reactionsSubject$ = new BehaviorSubject({}); - const remoteParticipants$ = of([aliceParticipant]); - const livekitRoom = mockLivekitRoom( - { localParticipant }, - { remoteParticipants$ }, - ); + // const remoteParticipants$ = of([aliceParticipant]); + const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, matrixRoom, - livekitRoom, mockMediaDevices({}), + mockMuteStates(), { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, ...callViewModelOptions, }, - constant(ConnectionState.Connected), handRaisedSubject$, reactionsSubject$, + of({ processor: undefined, supported: false }), ); return { vm, diff --git a/src/utils/test.ts b/src/utils/test.ts index 53b6d0ee..4aebd5fd 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -53,7 +53,7 @@ import { Config } from "../config/Config"; import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; -import { Handler, MuteStates } from "../state/MuteStates"; +import { MuteStates } from "../state/MuteStates"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); From f99a256c8616eba681bcb8fc0954bfd9fcc2fbef Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 13:53:39 -0400 Subject: [PATCH 034/144] Reset matrix-js-sdk to multi SFU branch --- package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 18877823..91583023 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 668706b4..6d0feed3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7544,7 +7544,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10297,9 +10297,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 37.13.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2f1d654f14be8dd03896e9e76f12017b6f9eec1c" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=b61e39a81458fb02d76d384e9c4bbef30fcd516a" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0" @@ -10315,9 +10315,9 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/ecd019c677c272c5598617dcde407dbe4b1b11460863b2a577e33f3fd8732c9d9073ec0221b471ec1eb24e2839eec20728db7f92c9348be83126547286e50805 + checksum: 10c0/da28671be560d3ef56e5a5465d1793dd6c2ccefcf68b510726f21b0a62107b115d40c1940e2a9bb8b915abdc600a9a84b248175ccdd9c77a3b313733efb8f497 languageName: node - linkType: soft + linkType: hard "matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": version: 1.13.1 From edd3eb874786e5cbc1ffe4171745cbfc4ed45066 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 13:54:54 -0400 Subject: [PATCH 035/144] Implement screen sharing --- src/room/InCallView.tsx | 23 ++++----------- src/state/CallViewModel.ts | 59 +++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 157ee46a..14f18fb7 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -116,8 +116,6 @@ import ringtoneOgg from "../sound/ringtone.ogg?url"; import { ConnectionLostError } from "../utils/errors.ts"; import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; -const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); - const maxTapDurationMs = 400; export interface ActiveCallProps @@ -224,7 +222,7 @@ export const InCallView: FC = ({ // Merge the refs so they can attach to the same element const containerRef = useMergedRefs(containerRef1, containerRef2); - const { hideScreensharing, showControls } = useUrlParams(); + const { showControls } = useUrlParams(); const muteAllAudio = useBehavior(muteAllAudio$); // Call pickup state and display names are needed for waiting overlay/sounds @@ -299,6 +297,7 @@ export const InCallView: FC = ({ const showFooter = useBehavior(vm.showFooter$); const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); + const sharingScreen = useBehavior(vm.sharingScreen$); // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; @@ -742,18 +741,6 @@ export const InCallView: FC = ({ const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const memberships = useBehavior(vm.memberships$); - const toggleScreensharing = useCallback(() => { - // TODO-MULTI-SFU implement screensharing - throw new Error("TODO-MULTI-SFU"); - // localParticipant - // .setScreenShareEnabled(!isScreenShareEnabled, { - // audio: true, - // selfBrowserSurface: "include", - // surfaceSwitching: "include", - // systemAudio: "include", - // }) - // .catch(logger.error); - }, []); const buttons: JSX.Element[] = []; @@ -775,13 +762,13 @@ export const InCallView: FC = ({ data-testid="incall_videomute" />, ); - if (canScreenshare && !hideScreensharing) { + if (vm.toggleScreenSharing !== null) { buttons.push( , diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 40828357..af750a9b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -15,6 +15,7 @@ import { type LocalParticipant, ParticipantEvent, type RemoteParticipant, + type Participant, } from "livekit-client"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { @@ -341,18 +342,7 @@ class UserMedia { this.presenter$ = this.scope.behavior( this.participant$.pipe( - switchMap( - (p) => - (p && - observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled))) ?? - of(false), - ), + switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))), ), ); } @@ -433,7 +423,19 @@ function getRoomMemberFromRtcMember( return { id, member }; } +function sharingScreen$(p: Participant): Observable { + return observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} + export class CallViewModel extends ViewModel { + private readonly urlParams = getUrlParams(); + private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); private readonly livekitE2EEKeyProvider = getE2eeKeyProvider( @@ -1850,6 +1852,37 @@ export class CallViewModel extends ViewModel { filter((v) => v.playSounds), ); + /** + * Whether we are sharing our screen. + */ + public readonly sharingScreen$ = this.scope.behavior( + from(this.localConnection).pipe( + switchMap((c) => sharingScreen$(c.livekitRoom.localParticipant)), + startWith(false), + ), + ); + + /** + * Callback for toggling screen sharing. If null, screen sharing is not + * available. + */ + public readonly toggleScreenSharing = + "getDisplayMedia" in (navigator.mediaDevices ?? {}) && + !this.urlParams.hideScreensharing + ? (): void => + void this.localConnection.then( + (c) => + void c.livekitRoom.localParticipant + .setScreenShareEnabled(!this.sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error), + ) + : null; + public constructor( // A call is permanently tied to a single Matrix room private readonly matrixRTCSession: MatrixRTCSession, @@ -1913,7 +1946,7 @@ export class CallViewModel extends ViewModel { }); this.leave$.pipe(this.scope.bind()).subscribe((reason) => { - const { confineToRoom } = getUrlParams(); + const { confineToRoom } = this.urlParams; leaveRTCSession(this.matrixRTCSession, "user") // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. .then(() => { From 6cf020763e4c1ad23f0fda75ec9e7f7cc9d72d98 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 21:26:16 -0400 Subject: [PATCH 036/144] Make UI react instantly to hanging up but also wait for leave sound This ensures that we don't see a mistaken 'reconnecting' toast while we're hanging up (and also that the leave sound gets a chance to play in widgets once again). --- src/room/GroupCallView.tsx | 76 ++++++++++++++++++------------ src/room/InCallView.tsx | 4 +- src/state/CallViewModel.ts | 95 +++++++++++++++++++------------------- 3 files changed, 95 insertions(+), 80 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 614c7b50..49d8b60b 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -41,7 +41,6 @@ import { ActiveCall } from "./InCallView"; import { type MuteStates } from "../state/MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; -import { leaveRTCSession } from "../rtcSessionHelpers"; import { saveKeyForRoom, useRoomEncryptionSystem, @@ -50,7 +49,12 @@ import { useRoomAvatar } from "./useRoomAvatar"; import { useRoomName } from "./useRoomName"; import { useJoinRule } from "./useJoinRule"; import { InviteModal } from "./InviteModal"; -import { HeaderStyle, type UrlParams, useUrlParams } from "../UrlParams"; +import { + getUrlParams, + HeaderStyle, + type UrlParams, + useUrlParams, +} from "../UrlParams"; import { E2eeType } from "../e2ee/e2eeType"; import { useAudioContext } from "../useAudioContext"; import { @@ -322,37 +326,62 @@ export const GroupCallView: FC = ({ setJoined(false); setLeft(true); const audioPromise = leaveSoundContext.current?.playSound(playSound); - // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, - // therefore we want the event to be sent instantly without getting queued/batched. - const sendInstantly = !!widget; - // we need to wait until the callEnded event is tracked on posthog. - // Otherwise the iFrame gets killed before the callEnded event got tracked. + // We need to wait until the callEnded event is tracked on PostHog, + // otherwise the iframe may get killed first. const posthogRequest = new Promise((resolve) => { + // To increase the likelihood of the PostHog event being sent out in + // widget mode before the iframe is killed, we ask it to skip the + // usual queuing/batching of requests. + const sendInstantly = widget !== null; PosthogAnalytics.instance.eventCallEnded.track( room.roomId, rtcSession.memberships.length, sendInstantly, - rtcSession, ); + // Unfortunately the PostHog library provides no way to await the + // tracking of an event, but we don't really want it to hold up the + // closing of the widget that long anyway, so giving it 10 ms will do. window.setTimeout(resolve, 10); }); void Promise.all([audioPromise, posthogRequest]) - .then(() => { + .catch((e) => + logger.error( + "Failed to play leave audio and/or send PostHog leave event", + e, + ), + ) + .then(async () => { if ( !isPasswordlessUser && !confineToRoom && !PosthogAnalytics.instance.isEnabled() - ) { + ) void navigate("/"); + + if (widget) { + // After this point the iframe could die at any moment! + try { + await widget.api.setAlwaysOnScreen(false); + } catch (e) { + logger.error( + "Failed to set call widget `alwaysOnScreen` to false", + e, + ); + } + // On a normal user hangup we can shut down and close the widget. But if an + // error occurs we should keep the widget open until the user reads it. + if (reason === "user" && !getUrlParams().returnToLobby) { + try { + await widget.api.transport.send(ElementWidgetActions.Close, {}); + } catch (e) { + logger.error("Failed to send close action", e); + } + widget.api.transport.stop(); + } } - }) - .catch(() => - logger.error( - "could failed to play leave audio or send posthog leave event", - ), - ); + }); }, [ setJoined, @@ -367,24 +396,11 @@ export const GroupCallView: FC = ({ ); useEffect(() => { - if (widget && joined) { + if (widget && joined) // set widget to sticky once joined. widget.api.setAlwaysOnScreen(true).catch((e) => { logger.error("Error calling setAlwaysOnScreen(true)", e); }); - - const onHangup = (ev: CustomEvent): void => { - widget.api.transport.reply(ev.detail, {}); - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - leaveRTCSession(rtcSession, "user").catch((e) => { - logger.error("Failed to leave RTC session", e); - }); - }; - widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup); - return (): void => { - widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); - }; - } }, [widget, joined, rtcSession]); const joinRule = useJoinRule(room); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 14f18fb7..20d53b3a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -151,7 +151,7 @@ export const ActiveCall: FC = (props) => { ); setVm(vm); - const sub = vm.left$.subscribe(props.onLeft); + const sub = vm.leave$.subscribe(props.onLeft); return (): void => { vm.destroy(); sub.unsubscribe(); @@ -798,7 +798,7 @@ export const InCallView: FC = ({ (); - - public leave(): void { - this.leave$.next("user"); - } - - private readonly _left$ = new Subject< - "decline" | "timeout" | "user" | "allOthersLeft" - >(); - public left$ = this._left$.asObservable(); - private readonly connectionInstructions$ = this.join$.pipe( switchMap(() => this.remoteConnections$), startWith(new Map()), @@ -1154,7 +1142,7 @@ export class CallViewModel extends ViewModel { ); private readonly allOthersLeft$ = this.matrixUserChanges$.pipe( - map(({ userIds, leftUserIds }) => { + filter(({ userIds, leftUserIds }) => { if (!this.userId) { logger.warn("Could not access user ID to compute allOthersLeft"); return false; @@ -1163,12 +1151,40 @@ export class CallViewModel extends ViewModel { userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 ); }), - startWith(false), + map(() => "allOthersLeft" as const), ); - public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft - ? this.allOthersLeft$ - : NEVER; + // Public for testing + public readonly autoLeave$ = merge( + this.options.autoLeaveWhenOthersLeft ? this.allOthersLeft$ : NEVER, + this.callPickupState$.pipe( + filter((state) => state === "timeout" || state === "decline"), + ), + ); + + private readonly userHangup$ = new Subject(); + public hangup(): void { + this.userHangup$.next(); + } + + private readonly widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable<[CustomEvent]> + ).pipe(tap(([ev]) => widget!.api.transport.reply(ev.detail, {}))); + + public readonly leave$: Observable< + "user" | "timeout" | "decline" | "allOthersLeft" + > = merge( + this.autoLeave$, + merge(this.userHangup$, this.widgetHangup$).pipe( + map(() => "user" as const), + ), + ); /** * List of MediaItems that we want to display, that are of type ScreenShare @@ -1929,34 +1945,17 @@ export class CallViewModel extends ViewModel { ); }); - this.allOthersLeft$ - .pipe( - this.scope.bind(), - filter((l) => (l && this.options.autoLeaveWhenOthersLeft) ?? false), - distinctUntilChanged(), - ) - .subscribe(() => { - this.leave$.next("allOthersLeft"); - }); - - this.callPickupState$.pipe(this.scope.bind()).subscribe((state) => { - if (state === "timeout" || state === "decline") { - this.leave$.next(state); - } - }); - - this.leave$.pipe(this.scope.bind()).subscribe((reason) => { - const { confineToRoom } = this.urlParams; - leaveRTCSession(this.matrixRTCSession, "user") - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - .then(() => { - if (!confineToRoom && !PosthogAnalytics.instance.isEnabled()) { - this._left$.next(reason); - } - }) - .catch((e) => { - logger.error("Error leaving RTC session", e); - }); + this.leave$.pipe(this.scope.bind()).subscribe(() => { + // Only sends Matrix leave event. The LiveKit session will disconnect once, uh... + // (TODO-MULTI-SFU does anything actually cause it to disconnect?) + void this.matrixRTCSession + .leaveRoomSession() + .catch((e) => logger.error("Error leaving RTC session", e)) + .then(async () => + widget?.api.transport + .send(ElementWidgetActions.HangupCall, {}) + .catch((e) => logger.error("Failed to send hangup action", e)), + ); }); // Pause upstream of all local media tracks when we're disconnected from From 530fbaf90afba3daf459d6df5e5264b9859a6409 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 21:39:36 -0400 Subject: [PATCH 037/144] Clear up the room membership confusion around reading session members --- src/state/CallViewModel.ts | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 867e5e09..5f365097 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -71,6 +71,7 @@ import { MembershipManagerEvent, Status, } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; import { ViewModel } from "./ViewModel"; import { @@ -127,7 +128,6 @@ import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; -import { IWidgetApiRequest } from "matrix-widget-api"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -475,9 +475,11 @@ export class CallViewModel extends ViewModel { ), ); - // TODO-MULTI-SFU make sure that we consider the room memberships here as well (so that here we only have valid memberships) - // this also makes it possible to use this memberships$ list in all observables based on it. - // there should be no other call to: this.matrixRTCSession.memberships! + /** + * The MatrixRTC session participants. + */ + // Note that MatrixRTCSession already filters the call memberships by users + // that are joined to the room; we don't need to perform extra filtering here. public readonly memberships$ = this.scope.behavior( fromEvent( this.matrixRTCSession, @@ -735,19 +737,17 @@ export class CallViewModel extends ViewModel { // than on Chrome/Firefox). This means it is important that we multicast the result so that we // don't do this work more times than we need to. This is achieved by converting to a behavior: public readonly memberDisplaynames$ = this.scope.behavior( - merge( - // Handle call membership changes. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ), - // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRoom, RoomStateEvent.Members), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ).pipe( - startWith(null), - map(() => { - const memberships = this.matrixRTCSession.memberships; + combineLatest( + [ + // Handle call membership changes + this.memberships$, + // Additionally handle display name changes (implicitly reacting to them) + fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe( + startWith(null), + ), + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ], + (memberships, _displaynames) => { const displaynameMap = new Map([ ["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName], ]); @@ -771,7 +771,7 @@ export class CallViewModel extends ViewModel { ); } return displaynameMap; - }), + }, ), ); From 0759f9b27d1e277344605e674413be4cfb2901ef Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 25 Sep 2025 21:29:02 -0400 Subject: [PATCH 038/144] Don't render audio from participants that aren't meant to be publishing --- src/livekit/MatrixAudioRenderer.tsx | 47 +++++---- src/room/InCallView.tsx | 14 +-- src/state/CallViewModel.ts | 150 +++++++++++++++------------- src/state/Connection.ts | 9 +- 4 files changed, 122 insertions(+), 98 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 0b87b27b..f402b32d 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { type Room as LivekitRoom } from "livekit-client"; +import { type Room as LivekitRoom, type Participant } from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { @@ -14,7 +14,7 @@ import { AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; @@ -22,13 +22,20 @@ import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; import {} from "@livekit/components-core"; export interface MatrixAudioRendererProps { + /** + * The service URL of the LiveKit room. + */ + url: string; + livekitRoom: LivekitRoom; /** * The list of participants to render audio for. * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ - members: CallMembership[]; - livekitRoom: LivekitRoom; + participants: { + participant: Participant; + member: RoomMember; + }[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -52,14 +59,14 @@ export interface MatrixAudioRendererProps { * @public */ export function LivekitRoomAudioRenderer({ - members, - muted, + url, livekitRoom, + participants, + muted, }: MatrixAudioRendererProps): ReactNode { - const validIdentities = useMemo( - () => - new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)), - [members], + const participantSet = useMemo( + () => new Set(participants.map(({ participant }) => participant)), + [participants], ); const loggedInvalidIdentities = useRef(new Set()); @@ -71,11 +78,11 @@ export function LivekitRoomAudioRenderer({ * @param identity The identity of the track that is invalid * @param validIdentities The list of valid identities */ - const logInvalid = (identity: string, validIdentities: Set): void => { + const logInvalid = (identity: string): void => { if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( - `[MatrixAudioRenderer] Audio track ${identity} has no matching matrix call member`, - `current members: ${Array.from(validIdentities.values())}`, + `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, + `current members: ${participants.map((p) => p.participant.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); @@ -93,23 +100,27 @@ export function LivekitRoomAudioRenderer({ room: livekitRoom, }, ).filter((ref) => { - const isValid = validIdentities?.has(ref.participant.identity); + const isValid = participantSet?.has(ref.participant); if (!isValid && !ref.participant.isLocal) - logInvalid(ref.participant.identity, validIdentities); + logInvalid(ref.participant.identity); return ( !ref.participant.isLocal && ref.publication.kind === Track.Kind.Audio && isValid ); }); + useEffect(() => { - if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) { + if ( + loggedInvalidIdentities.current.size && + tracks.every((t) => participantSet.has(t.participant)) + ) { logger.debug( - `[MatrixAudioRenderer] All audio tracks have a matching matrix call member identity.`, + `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, ); loggedInvalidIdentities.current.clear(); } - }, [tracks, validIdentities]); + }, [tracks, participantSet, url]); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 20d53b3a..db2c0f2a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -286,6 +286,8 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const allLivekitRooms = useBehavior(vm.allLivekitRooms$); + const participantsByRoom = useBehavior(vm.participantsByRoom$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -739,9 +741,6 @@ export const InCallView: FC = ({ matrixRoom.roomId, ); - const allLivekitRooms = useBehavior(vm.allLivekitRooms$); - const memberships = useBehavior(vm.memberships$); - const buttons: JSX.Element[] = []; buttons.push( @@ -862,11 +861,12 @@ export const InCallView: FC = ({ ) } - {allLivekitRooms.map((roomItem) => ( + {participantsByRoom.map(({ livekitRoom, url, participants }) => ( ))} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3b1cdc96..3dff08d3 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -480,7 +480,7 @@ export class CallViewModel extends ViewModel { */ // Note that MatrixRTCSession already filters the call memberships by users // that are joined to the room; we don't need to perform extra filtering here. - public readonly memberships$ = this.scope.behavior( + private readonly memberships$ = this.scope.behavior( fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, @@ -679,16 +679,19 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - private readonly participants$ = this.scope.behavior< + public readonly participantsByRoom$ = this.scope.behavior< { - participant: LocalParticipant | RemoteParticipant; - member: RoomMember; livekitRoom: LivekitRoom; + url: string; + participants: { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + }[]; }[] >( - from(this.localConnection) + combineLatest([this.localConnection, this.localFocus]) .pipe( - switchMap((localConnection) => { + switchMap(([localConnection, localFocus]) => { const memberError = (): never => { throw new Error("No room member for call membership"); }; @@ -696,32 +699,41 @@ export class CallViewModel extends ViewModel { participant: localConnection.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), - livekitRoom: localConnection.livekitRoom, }; + return this.remoteConnections$.pipe( switchMap((connections) => combineLatest( - [localConnection, ...connections.values()].map((c) => + [ + [localFocus.livekit_service_url, localConnection] as const, + ...connections, + ].map(([url, c]) => c.publishingParticipants$.pipe( - map((ps) => - ps.map(({ participant, membership }) => ({ + map((ps) => { + const participants: { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + }[] = ps.map(({ participant, membership }) => ({ participant, member: getRoomMemberFromRtcMember( membership, this.matrixRoom, )?.member ?? memberError(), + })); + if (c === localConnection) + participants.push(localParticipant); + + return { livekitRoom: c.livekitRoom, - })), - ), + url, + participants, + }; + }), ), ), ), ), - map((remoteParticipants) => [ - localParticipant, - ...remoteParticipants.flat(1), - ]), ); }), ) @@ -798,7 +810,7 @@ export class CallViewModel extends ViewModel { */ private readonly mediaItems$ = this.scope.behavior( combineLatest([ - this.participants$, + this.participantsByRoom$, duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -806,71 +818,75 @@ export class CallViewModel extends ViewModel { scan( ( prevItems, - [participants, duplicateTiles, memberships, showNonMemberTiles], + [participantsByRoom, duplicateTiles, memberships, showNonMemberTiles], ) => { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const { participant, member, livekitRoom } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { - prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } - } - yield [ - mediaId, - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see above) - prevMedia ?? - new UserMedia( - mediaId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), - ), - ), - ]; + for (const { livekitRoom, participants } of participantsByRoom) { + for (const { participant, member } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; + + for (let i = 0; i < 1 + duplicateTiles; i++) { + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } + } - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, + mediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? + new UserMedia( + mediaId, member, participant, this.options.encryptionSystem, livekitRoom, + this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixId) ?? "[👻]"), ), + this.handsRaised$.pipe( + map((v) => v[matrixId]?.time ?? null), + ), + this.reactions$.pipe( + map((v) => v[matrixId] ?? undefined), + ), ), ]; + + if (participant?.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, + member, + participant, + this.options.encryptionSystem, + livekitRoom, + this.pretendToBeDisconnected$, + this.memberDisplaynames$.pipe( + map((m) => m.get(matrixId) ?? "[👻]"), + ), + ), + ]; + } } } } diff --git a/src/state/Connection.ts b/src/state/Connection.ts index e14ddc9a..2513382c 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -93,11 +93,9 @@ export class Connection { ); this.publishingParticipants$ = this.scope.behavior( - combineLatest([ - this.participantsIncludingSubscribers$, - this.membershipsFocusMap$, - ]).pipe( - map(([participants, membershipsFocusMap]) => + combineLatest( + [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], + (participants, membershipsFocusMap) => membershipsFocusMap // Find all members that claim to publish on this connection .flatMap(({ membership, focus }) => @@ -113,7 +111,6 @@ export class Connection { ); return participant ? [{ participant, membership }] : []; }), - ), ), [], ); From dbdf853d558dc0342cf68098cf09ced93ab18f9e Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 26 Sep 2025 13:20:55 -0400 Subject: [PATCH 039/144] Stop connections on view model destroy --- src/state/CallViewModel.ts | 1 - src/state/Connection.ts | 13 +++++++------ src/state/ObservableScope.ts | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3dff08d3..439d2662 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -359,7 +359,6 @@ class UserMedia { public destroy(): void { this.scope.end(); - this.vm.destroy(); } } diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 2513382c..db456ba0 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -55,6 +55,7 @@ export class Connection { } public stop(): void { + if (this.stopped) return; void this.livekitRoom.disconnect(); this.stopped = true; } @@ -117,6 +118,8 @@ export class Connection { this.connectionState$ = this.scope.behavior( connectionStateObserver(this.livekitRoom), ); + + this.scope.onEnd(() => this.stop()); } } @@ -137,11 +140,6 @@ export class PublishConnection extends Connection { } } - public stop(): void { - void this.livekitRoom.disconnect(); - this.stopped = true; - } - public constructor( focus: LivekitFocus, livekitAlias: string, @@ -220,7 +218,10 @@ export class PublishConnection extends Connection { } return this.livekitRoom.localParticipant.isCameraEnabled; }); - // TODO-MULTI-SFU: Unset mute state handlers on destroy + this.scope.onEnd(() => { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + }); const syncDevice = ( kind: MediaDeviceKind, diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 1cddfbff..fe99d89b 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -36,11 +36,16 @@ export class ObservableScope { return this.bindImpl; } - private readonly shareImpl: MonoTypeOperator = share({ resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false }) + private readonly shareImpl: MonoTypeOperator = share({ + resetOnError: false, + resetOnComplete: false, + resetOnRefCountZero: false, + }); /** * Shares (multicasts) the Observable as a hot Observable. */ - public readonly share: MonoTypeOperator = (input$) => input$.pipe(this.bindImpl, this.shareImpl) + public readonly share: MonoTypeOperator = (input$) => + input$.pipe(this.bindImpl, this.shareImpl); /** * Converts an Observable to a Behavior. If no initial value is specified, the @@ -76,6 +81,13 @@ export class ObservableScope { this.ended$.next(); this.ended$.complete(); } + + /** + * Register a callback to be executed when the scope is ended. + */ + public onEnd(callback: () => void): void { + this.ended$.subscribe(callback); + } } /** From a4a0a58a72d1fe034fc0e53669ae375fc8102ee8 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 26 Sep 2025 13:26:42 -0400 Subject: [PATCH 040/144] Remove the option to show non-member ("ghost") participants As we'd like to get the multi-SFU feature branch shipped, this is not the most important debugging tool to expend effort on at the moment. --- locales/en/app.json | 1 - src/settings/DeveloperSettingsTab.tsx | 18 --- src/settings/settings.ts | 4 - src/state/CallViewModel.test.ts | 48 ------- src/state/CallViewModel.ts | 187 ++++++++------------------ 5 files changed, 59 insertions(+), 199 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 007e372a..dc027c92 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,7 +74,6 @@ "matrix_id": "Matrix ID: {{id}}", "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", - "show_non_member_tiles": "Show tiles for non-member media", "url_params": "URL parameters", "use_new_membership_manager": "Use the new implementation of the call MembershipManager", "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key" diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index d503385b..1949ecf7 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -13,7 +13,6 @@ import { useSetting, duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, - showNonMemberTiles as showNonMemberTilesSetting, showConnectionStats as showConnectionStatsSetting, useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, @@ -35,9 +34,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { const [debugTileLayout, setDebugTileLayout] = useSetting( debugTileLayoutSetting, ); - const [showNonMemberTiles, setShowNonMemberTiles] = useSetting( - showNonMemberTilesSetting, - ); const [showConnectionStats, setShowConnectionStats] = useSetting( showConnectionStatsSetting, @@ -128,20 +124,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { } /> - - ): void => { - setShowNonMemberTiles(event.target.checked); - }, - [setShowNonMemberTiles], - )} - /> - ( - "show-non-member-tiles", - false, -); export const debugTileLayout = new Setting("debug-tile-layout", false); export const showConnectionStats = new Setting( diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index b736b780..07c78ef6 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -69,7 +69,6 @@ import { } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; import type { RaisedHandInfo } from "../reactions"; -import { showNonMemberTiles } from "../settings/settings"; import { alice, aliceDoppelganger, @@ -824,53 +823,6 @@ test("participants must have a MatrixRTCSession to be visible", () => { }); }); -test("shows participants without MatrixRTCSession when enabled in settings", () => { - try { - // enable the setting: - showNonMemberTiles.setValue(true); - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = " abc"; - const expectedLayoutMarbles = "abc"; - - withCallViewModel( - { - remoteParticipants$: behavior(scenarioInputMarbles, { - a: [], - b: [aliceParticipant], - c: [aliceParticipant, bobParticipant], - }), - rtcMembers$: constant([localRtcMember]), // No one else joins the MatrixRTC session - }, - (vm) => { - vm.setGridMode("grid"); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: ["local:0"], - }, - b: { - type: "one-on-one", - local: "local:0", - remote: `${aliceId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - }, - ); - }, - ); - }); - } finally { - showNonMemberTiles.setValue(showNonMemberTiles.defaultValue); - } -}); - it("should show at least one tile per MatrixRTCSession", () => { withTestScheduler(({ behavior, expectObservable }) => { // iterate through some combinations of MatrixRTC memberships diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 439d2662..7e3a5bdf 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -92,7 +92,6 @@ import { duplicateTiles, playReactionsSound, showReactions, - showNonMemberTiles, } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled$ } from "../controls"; @@ -812,152 +811,84 @@ export class CallViewModel extends ViewModel { this.participantsByRoom$, duplicateTiles.value$, this.memberships$, - showNonMemberTiles.value$, ]).pipe( - scan( - ( - prevItems, - [participantsByRoom, duplicateTiles, memberships, showNonMemberTiles], - ) => { - const newItems: Map = new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const { livekitRoom, participants } of participantsByRoom) { - for (const { participant, member } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; + scan((prevItems, [participantsByRoom, duplicateTiles, memberships]) => { + const newItems: Map = new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const { livekitRoom, participants } of participantsByRoom) { + for (const { participant, member } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { - prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } + for (let i = 0; i < 1 + duplicateTiles; i++) { + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; } + } + yield [ + mediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? + new UserMedia( + mediaId, + member, + participant, + this.options.encryptionSystem, + livekitRoom, + this.mediaDevices, + this.pretendToBeDisconnected$, + this.memberDisplaynames$.pipe( + map((m) => m.get(matrixId) ?? "[👻]"), + ), + this.handsRaised$.pipe( + map((v) => v[matrixId]?.time ?? null), + ), + this.reactions$.pipe( + map((v) => v[matrixId] ?? undefined), + ), + ), + ]; + + if (participant?.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; yield [ - mediaId, - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see above) - prevMedia ?? - new UserMedia( - mediaId, + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, member, participant, this.options.encryptionSystem, livekitRoom, - this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixId) ?? "[👻]"), ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), - ), ), ]; - - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; - yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - ), - ]; - } } } } - }.bind(this)(), - ); + } + }.bind(this)(), + ); - // Generate non member items (items without a corresponding MatrixRTC member) - // Those items should not be rendered, they are participants in LiveKit that do not have a corresponding - // MatrixRTC members. This cannot be any good: - // - A malicious user impersonates someone - // - Someone injects abusive content - // - The user cannot have encryption keys so it makes no sense to participate - // We can only trust users that have a MatrixRTC member event. - // - // This is still available as a debug option. This can be useful - // - If one wants to test scalability using the LiveKit CLI. - // - If an experimental project does not yet do the MatrixRTC bits. - // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. - // TODO-MULTI-SFU - // const newNonMemberItems = showNonMemberTiles - // ? new Map( - // function* ( - // this: CallViewModel, - // ): Iterable<[string, MediaItem]> { - // for (const participant of remoteParticipants) { - // for (let i = 0; i < 1 + duplicateTiles; i++) { - // const maybeNonMemberParticipantId = - // participant.identity + ":" + i; - // if (!newItems.has(maybeNonMemberParticipantId)) { - // const nonMemberId = maybeNonMemberParticipantId; - // yield [ - // nonMemberId, - // prevItems.get(nonMemberId) ?? - // new UserMedia( - // nonMemberId, - // undefined, - // participant, - // this.options.encryptionSystem, - // localConnection.livekitRoom, - // this.mediaDevices, - // this.pretendToBeDisconnected$, - // this.memberDisplaynames$.pipe( - // map( - // (m) => - // m.get(participant.identity) ?? "[👻]", - // ), - // ), - // of(null), - // of(null), - // ), - // ]; - // } - // } - // } - // }.bind(this)(), - // ) - // : new Map(); - // if (newNonMemberItems.size > 0) { - // logger.debug("Added NonMember items: ", newNonMemberItems); - // } - - const combinedNew = new Map([ - // ...newNonMemberItems.entries(), - ...newItems.entries(), - ]); - - for (const [id, t] of prevItems) - if (!combinedNew.has(id)) t.destroy(); - return combinedNew; - }, - new Map(), - ), + for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); + return newItems; + }, new Map()), map((mediaItems) => [...mediaItems.values()]), finalizeValue((ts) => { for (const t of ts) t.destroy(); From edf68d16b748d230c5651d06b39f4836bdfd17cc Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 30 Sep 2025 11:33:45 +0200 Subject: [PATCH 041/144] refactoring: prep work extract to file + documentation --- src/state/CallViewModel.ts | 3 +- src/state/Connection.ts | 281 ++++++++------------------------- src/state/PublishConnection.ts | 224 ++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 213 deletions(-) create mode 100644 src/state/PublishConnection.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7e3a5bdf..b6327cfa 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -122,11 +122,12 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { Connection, PublishConnection } from "./Connection"; +import { Connection } from "./Connection"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; +import { PublishConnection } from "./PublishConnection.ts"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; diff --git a/src/state/Connection.ts b/src/state/Connection.ts index db456ba0..f725ddda 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -5,55 +5,50 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - connectedParticipantsObserver, - connectionStateObserver, -} from "@livekit/components-core"; -import { - ConnectionState, - Room as LivekitRoom, - type E2EEOptions, - Track, - LocalVideoTrack, -} from "livekit-client"; +import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; +import { type ConnectionState, type E2EEOptions, Room as LivekitRoom } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; -import { - type LivekitFocus, - type CallMembership, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - NEVER, - type Observable, - type Subscription, - switchMap, -} from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest } from "rxjs"; -import { type SelectedDevice, type MediaDevices } from "./MediaDevices"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; -import { getValue } from "../utils/observable"; -import { getUrlParams } from "../UrlParams"; -import { type MuteStates } from "./MuteStates"; -import { - type ProcessorState, - trackProcessorSync, -} from "../livekit/TrackProcessorContext"; -import { observeTrackReference$ } from "./MediaViewModel"; +/** + * A connection to a Matrix RTC LiveKit backend. + * + * Expose observables for participants and connection state. + */ export class Connection { + + /** + * Whether the connection has been stopped. + * @see Connection.stop + * */ protected stopped = false; + /** + * Starts the connection. + * + * This will: + * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + * 3. Connect to the configured LiveKit room. + */ public async start(): Promise { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); } + /** + * Stops the connection. + * + * This will disconnect from the LiveKit room. + * If the connection is already stopped, this is a no-op. + */ public stop(): void { if (this.stopped) return; void this.livekitRoom.disconnect(); @@ -63,16 +58,47 @@ export class Connection { protected readonly sfuConfig = getSFUConfigWithOpenID( this.client, this.focus.livekit_service_url, - this.livekitAlias, + this.focus.livekit_alias ); - public readonly participantsIncludingSubscribers$; + /* + * An observable of the participants in the livekit room, including subscribers. + * Converts the livekit room events ParticipantConnected/ParticipantDisconnected/StateChange to an observable. + */ + protected readonly participantsIncludingSubscribers$; + + /** + * An observable of the participants that are publishing on this connection. + * This is derived from `participantsIncludingSubscribers$` and `membershipsFocusMap$`. + * It filters the participants to only those that are associated with a membership that claims to publish on this connection. + */ public readonly publishingParticipants$; + + /** + * The LiveKit room instance. + */ public readonly livekitRoom: LivekitRoom; + /** + * An observable of the livekit connection state. + * Converts the livekit room events StateChange to an observable. + */ public connectionState$: Behavior; + + /** + * Creates a new connection to a matrix RTC LiveKit backend. + * + * @param livekitRoom - Optional LiveKit room instance to use. If not provided, a new instance will be created. + * @param focus - The focus server to connect to. + * @param livekitAlias - The livekit alias to use when connecting to the focus server. TODO duplicate of focus? + * @param client - The matrix client, used to fetch the OpenId token. TODO refactor to avoid passing the whole client + * @param scope - The observable scope to use for creating observables. + * @param membershipsFocusMap$ - The observable of the current call RTC memberships and their associated focus. + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. TODO refactor to avoid passing the whole options? + */ public constructor( protected readonly focus: LivekitFocus, + // TODO : remove livekitAlias, it's already in focus? protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, @@ -80,17 +106,17 @@ export class Connection { { membership: CallMembership; focus: LivekitFocus }[] >, e2eeLivekitOptions: E2EEOptions | undefined, - livekitRoom: LivekitRoom | undefined = undefined, + livekitRoom: LivekitRoom | undefined = undefined ) { this.livekitRoom = livekitRoom ?? new LivekitRoom({ ...defaultLiveKitOptions, - e2ee: e2eeLivekitOptions, + e2ee: e2eeLivekitOptions }); this.participantsIncludingSubscribers$ = this.scope.behavior( connectedParticipantsObserver(this.livekitRoom), - [], + [] ); this.publishingParticipants$ = this.scope.behavior( @@ -102,193 +128,24 @@ export class Connection { .flatMap(({ membership, focus }) => focus.livekit_service_url === this.focus.livekit_service_url ? [membership] - : [], + : [] ) // Find all associated publishing livekit participant objects .flatMap((membership) => { const participant = participants.find( (p) => - p.identity === `${membership.sender}:${membership.deviceId}`, + p.identity === `${membership.sender}:${membership.deviceId}` ); return participant ? [{ participant, membership }] : []; - }), + }) ), - [], + [] ); this.connectionState$ = this.scope.behavior( - connectionStateObserver(this.livekitRoom), + connectionStateObserver(this.livekitRoom) ); this.scope.onEnd(() => this.stop()); } } -export class PublishConnection extends Connection { - public async start(): Promise { - this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); - - if (!this.stopped) { - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value, - }); - for (const track of tracks) { - await this.livekitRoom.localParticipant.publishTrack(track); - } - } - } - - public constructor( - focus: LivekitFocus, - livekitAlias: string, - client: MatrixClient, - scope: ObservableScope, - membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, - devices: MediaDevices, - private readonly muteStates: MuteStates, - e2eeLivekitOptions: E2EEOptions | undefined, - trackerProcessorState$: Behavior, - ) { - logger.info("[LivekitRoom] Create LiveKit room"); - const { controlledAudioDevices } = getUrlParams(); - - const room = new LivekitRoom({ - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: trackerProcessorState$.value.processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee: e2eeLivekitOptions, - }); - room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - super( - focus, - livekitAlias, - client, - scope, - membershipsFocusMap$, - e2eeLivekitOptions, - room, - ); - - // Setup track processor syncing (blur) - const track$ = this.scope.behavior( - observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ), - ); - trackProcessorSync(track$, trackerProcessorState$); - - this.muteStates.audio.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); - } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; - }); - this.muteStates.video.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); - } - return this.livekitRoom.localParticipant.isCameraEnabled; - }); - this.scope.onEnd(() => { - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - }); - - const syncDevice = ( - kind: MediaDeviceKind, - selected$: Observable, - ): Subscription => - selected$.pipe(this.scope.bind()).subscribe((device) => { - if (this.connectionState$.value !== ConnectionState.Connected) return; - logger.info( - "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - this.livekitRoom.getActiveDevice(kind), - " !== ", - device?.id, - ); - if ( - device !== undefined && - this.livekitRoom.getActiveDevice(kind) !== device.id - ) { - this.livekitRoom - .switchActiveDevice(kind, device.id) - .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e), - ); - } - }); - - syncDevice("audioinput", devices.audioInput.selected$); - if (!controlledAudioDevices) - syncDevice("audiooutput", devices.audioOutput.selected$); - syncDevice("videoinput", devices.videoInput.selected$); - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. - devices.audioInput.selected$ - .pipe( - switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - this.scope.bind(), - ) - .subscribe(() => { - if (this.connectionState$.value !== ConnectionState.Connected) return; - const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values(), - ).find((d) => d.source === Track.Source.Microphone)?.track; - - if ( - activeMicTrack && - // only restart if the stream is still running: LiveKit will detect - // when a track stops & restart appropriately, so this is not our job. - // Plus, we need to avoid restarting again if the track is already in - // the process of being restarted. - activeMicTrack.mediaStreamTrack.readyState !== "ended" - ) { - // Restart the track, which will cause Livekit to do another - // getUserMedia() call with deviceId: default to get the *new* default device. - // Note that room.switchActiveDevice() won't work: Livekit will ignore it because - // the deviceId hasn't changed (was & still is default). - this.livekitRoom.localParticipant - .getTrackPublication(Track.Source.Microphone) - ?.audioTrack?.restartTrack() - .catch((e) => { - logger.error(`Failed to restart audio device track`, e); - }); - } - }); - } -} diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts new file mode 100644 index 00000000..532be26c --- /dev/null +++ b/src/state/PublishConnection.ts @@ -0,0 +1,224 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ +import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, Track } from "livekit-client"; +import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; + +import type { CallMembership, LivekitFocus } from "../../../matrix-js-sdk/lib/matrixrtc"; +import type { MatrixClient } from "../../../matrix-js-sdk"; +import type { ObservableScope } from "./ObservableScope.ts"; +import type { Behavior } from "./Behavior.ts"; +import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; +import type { MuteStates } from "./MuteStates.ts"; +import { type ProcessorState, trackProcessorSync } from "../livekit/TrackProcessorContext.tsx"; +import { logger } from "../../../matrix-js-sdk/lib/logger"; +import { getUrlParams } from "../UrlParams.ts"; +import { defaultLiveKitOptions } from "../livekit/options.ts"; +import { getValue } from "../utils/observable.ts"; +import { observeTrackReference$ } from "./MediaViewModel.ts"; +import { Connection } from "./Connection.ts"; + +/** + * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. + * This connection will publish the local user's audio and video tracks. + */ +export class PublishConnection extends Connection { + + + /** + * Start the connection to LiveKit and publish local tracks. + * + * This will: + * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + * 3. Connect to the configured LiveKit room. + * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + */ + public async start(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + + if (!this.stopped) { + // TODO this can throw errors? It will also prompt for permissions if not already granted + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: this.muteStates.audio.enabled$.value, + video: this.muteStates.video.enabled$.value + }); + for (const track of tracks) { + // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally + // with a timeout. + await this.livekitRoom.localParticipant.publishTrack(track); + // TODO: check if the connection is still active? and break the loop if not? + } + } + }; + + + /** + * Creates a new PublishConnection. + * @param focus - The Livekit focus object containing the configuration for the connection. + * @param livekitAlias - TODO: remove, use focus.livekit_alias instead + * @param client - The Matrix client to use for authentication. TODO: remove only pick OpenIDClientParts + * @param scope - The observable scope to use for managing subscriptions. + * @param membershipsFocusMap$ - An observable of the current RTC call memberships and their associated focus. + * @param devices - The media devices to use for audio and video input. + * @param muteStates - The mute states for audio and video. + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. + * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). + */ + public constructor( + focus: LivekitFocus, + livekitAlias: string, + client: MatrixClient, + scope: ObservableScope, + membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, + devices: MediaDevices, + private readonly muteStates: MuteStates, + e2eeLivekitOptions: E2EEOptions | undefined, + trackerProcessorState$: Behavior + ) { + logger.info("[LivekitRoom] Create LiveKit room"); + const { controlledAudioDevices } = getUrlParams(); + + const room = new LivekitRoom({ + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: trackerProcessorState$.value.processor + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id + }, + e2ee: e2eeLivekitOptions + }); + room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + + super( + focus, + livekitAlias, + client, + scope, + membershipsFocusMap$, + e2eeLivekitOptions, + room + ); + + // Setup track processor syncing (blur) + const track$ = this.scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( + map((trackRef) => { + const track = trackRef?.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }) + ) + ); + trackProcessorSync(track$, trackerProcessorState$); + + this.muteStates.audio.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit audio input mute state", e); + } + return this.livekitRoom.localParticipant.isMicrophoneEnabled; + }); + this.muteStates.video.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setCameraEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit video input mute state", e); + } + return this.livekitRoom.localParticipant.isCameraEnabled; + }); + this.scope.onEnd(() => { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + }); + + const syncDevice = ( + kind: MediaDeviceKind, + selected$: Observable + ): Subscription => + selected$.pipe(this.scope.bind()).subscribe((device) => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + logger.info( + "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", + this.livekitRoom.getActiveDevice(kind), + " !== ", + device?.id + ); + if ( + device !== undefined && + this.livekitRoom.getActiveDevice(kind) !== device.id + ) { + this.livekitRoom + .switchActiveDevice(kind, device.id) + .catch((e) => + logger.error(`Failed to sync ${kind} device with LiveKit`, e) + ); + } + }); + + syncDevice("audioinput", devices.audioInput.selected$); + if (!controlledAudioDevices) + syncDevice("audiooutput", devices.audioOutput.selected$); + syncDevice("videoinput", devices.videoInput.selected$); + // Restart the audio input track whenever we detect that the active media + // device has changed to refer to a different hardware device. We do this + // for the sake of Chrome, which provides a "default" device that is meant + // to match the system's default audio input, whatever that may be. + // This is special-cased for only audio inputs because we need to dig around + // in the LocalParticipant object for the track object and there's not a nice + // way to do that generically. There is usually no OS-level default video capture + // device anyway, and audio outputs work differently. + devices.audioInput.selected$ + .pipe( + switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), + this.scope.bind() + ) + .subscribe(() => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + const activeMicTrack = Array.from( + this.livekitRoom.localParticipant.audioTrackPublications.values() + ).find((d) => d.source === Track.Source.Microphone)?.track; + + if ( + activeMicTrack && + // only restart if the stream is still running: LiveKit will detect + // when a track stops & restart appropriately, so this is not our job. + // Plus, we need to avoid restarting again if the track is already in + // the process of being restarted. + activeMicTrack.mediaStreamTrack.readyState !== "ended" + ) { + // Restart the track, which will cause Livekit to do another + // getUserMedia() call with deviceId: default to get the *new* default device. + // Note that room.switchActiveDevice() won't work: Livekit will ignore it because + // the deviceId hasn't changed (was & still is default). + this.livekitRoom.localParticipant + .getTrackPublication(Track.Source.Microphone) + ?.audioTrack?.restartTrack() + .catch((e) => { + logger.error(`Failed to restart audio device track`, e); + }); + } + }); + } +} From 2819c7959013a6a287ebc9960f898a4b454cb4a1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 30 Sep 2025 16:47:45 +0200 Subject: [PATCH 042/144] use updated multi sfu js-sdk Signed-off-by: Timo K --- src/room/useActiveFocus.ts | 45 ----------------------- src/rtcSessionHelpers.ts | 75 +++++++++++++++++++------------------- src/state/CallViewModel.ts | 6 ++- src/state/Connection.ts | 10 ++--- yarn.lock | 4 +- 5 files changed, 48 insertions(+), 92 deletions(-) delete mode 100644 src/room/useActiveFocus.ts diff --git a/src/room/useActiveFocus.ts b/src/room/useActiveFocus.ts deleted file mode 100644 index 7a8f4521..00000000 --- a/src/room/useActiveFocus.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, -} from "matrix-js-sdk/lib/matrixrtc"; -import { useCallback, useRef } from "react"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitFocus, isLivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; - -import { useTypedEventEmitterState } from "../useEvents"; - -/** - * Gets the currently active (livekit) focus for a MatrixRTC session - * This logic is specific to livekit foci where the whole call must use one - * and the same focus. - */ -export function useActiveLivekitFocus( - rtcSession: MatrixRTCSession, -): LivekitFocus | undefined { - const prevActiveFocus = useRef(undefined); - return useTypedEventEmitterState( - rtcSession, - MatrixRTCSessionEvent.MembershipsChanged, - useCallback(() => { - const f = rtcSession.getActiveFocus(); - // Only handle foci with type="livekit" for now. - if (f && isLivekitFocus(f) && !deepCompare(f, prevActiveFocus.current)) { - const oldestMembership = rtcSession.getOldestMembership(); - logger.info( - `Got new active focus from membership: ${oldestMembership?.sender}/${oldestMembership?.deviceId}. - Updated focus (focus switch) from ${JSON.stringify(prevActiveFocus.current)} to ${JSON.stringify(f)}`, - ); - prevActiveFocus.current = f; - } - return prevActiveFocus.current; - }, [rtcSession]), - ); -} diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index b6918f3a..175b35f4 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -6,11 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { - isLivekitFocusConfig, - type LivekitFocusConfig, - type LivekitFocus, - type LivekitFocusSelection, type MatrixRTCSession, + isLivekitTransportConfig, + type LivekitTransportConfig, + type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; @@ -24,13 +23,6 @@ import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; -export function makeActiveFocus(): LivekitFocusSelection { - return { - type: "livekit", - focus_selection: "oldest_membership", - }; -} - export function getLivekitAlias(rtcSession: MatrixRTCSession): string { // For now we assume everything is a room-scoped call return rtcSession.room.roomId; @@ -38,13 +30,13 @@ export function getLivekitAlias(rtcSession: MatrixRTCSession): string { async function makeFocusInternal( rtcSession: MatrixRTCSession, -): Promise { +): Promise { logger.log("Searching for a preferred focus"); const livekitAlias = getLivekitAlias(rtcSession); const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth"); if (urlFromStorage !== null) { - const focusFromStorage: LivekitFocus = { + const focusFromStorage: LivekitTransport = { type: "livekit", livekit_service_url: urlFromStorage, livekit_alias: livekitAlias, @@ -57,7 +49,7 @@ async function makeFocusInternal( const domain = rtcSession.room.client.getDomain(); if (localStorage.getItem("timo-focus-url")) { const timoFocusUrl = localStorage.getItem("timo-focus-url")!; - const focusFromUrl: LivekitFocus = { + const focusFromUrl: LivekitTransport = { type: "livekit", livekit_service_url: timoFocusUrl, livekit_alias: livekitAlias, @@ -72,8 +64,8 @@ async function makeFocusInternal( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - const focus: LivekitFocusConfig | undefined = wellKnownFoci.find( - (f) => f && isLivekitFocusConfig(f), + const focus: LivekitTransportConfig | undefined = wellKnownFoci.find( + (f) => f && isLivekitTransportConfig(f), ); if (focus !== undefined) { logger.log("Using LiveKit focus from .well-known: ", focus); @@ -84,7 +76,7 @@ async function makeFocusInternal( const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { - const focusFromConf: LivekitFocus = { + const focusFromConf: LivekitTransport = { type: "livekit", livekit_service_url: urlFromConf, livekit_alias: livekitAlias, @@ -98,7 +90,7 @@ async function makeFocusInternal( export async function makeFocus( rtcSession: MatrixRTCSession, -): Promise { +): Promise { const focus = await makeFocusInternal(rtcSession); // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID( @@ -111,10 +103,11 @@ export async function makeFocus( export async function enterRTCSession( rtcSession: MatrixRTCSession, - focus: LivekitFocus, + focus: LivekitTransport, encryptMedia: boolean, useNewMembershipManager = true, useExperimentalToDeviceTransport = false, + useMultiSfu = true, ): Promise { PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); @@ -127,25 +120,31 @@ export async function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - rtcSession.joinRoomSession([focus], focus, { - notificationType, - callIntent, - useNewMembershipManager, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport, - }); + // Multi-sfu does not need a focus preferred list. just the focus that is actually used. + rtcSession.joinRoomSession( + useMultiSfu ? [focus] : [], + useMultiSfu ? focus : undefined, + { + notificationType, + callIntent, + useNewMembershipManager, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: + matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport, + }, + ); if (widget) { try { await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7e3a5bdf..2f4bfa0c 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -64,7 +64,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, - isLivekitFocus, + isLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -493,7 +493,9 @@ export class CallViewModel extends ViewModel { map((memberships) => memberships.flatMap((m) => { const f = this.matrixRTCSession.resolveActiveFocus(m); - return f && isLivekitFocus(f) ? [{ membership: m, focus: f }] : []; + return f && isLivekitTransport(f) + ? [{ membership: m, focus: f }] + : []; }), ), ), diff --git a/src/state/Connection.ts b/src/state/Connection.ts index db456ba0..8eaed463 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -18,7 +18,7 @@ import { } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { - type LivekitFocus, + type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; import { @@ -72,12 +72,12 @@ export class Connection { public connectionState$: Behavior; public constructor( - protected readonly focus: LivekitFocus, + protected readonly focus: LivekitTransport, protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] + { membership: CallMembership; focus: LivekitTransport }[] >, e2eeLivekitOptions: E2EEOptions | undefined, livekitRoom: LivekitRoom | undefined = undefined, @@ -141,12 +141,12 @@ export class PublishConnection extends Connection { } public constructor( - focus: LivekitFocus, + focus: LivekitTransport, livekitAlias: string, client: MatrixClient, scope: ObservableScope, membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] + { membership: CallMembership; focus: LivekitTransport }[] >, devices: MediaDevices, private readonly muteStates: MuteStates, diff --git a/yarn.lock b/yarn.lock index 4429b7d4..a149eaf5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10299,7 +10299,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=ca4a9c655537702daf9a69ed5d94831cebc49666" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=d94d02d19b9f17c724b5919b185fea3413dbf7a2" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10315,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/1fb0933d0bb686b0f290b1a62f75eec290b7c52a410d5968c2ccfb527a64e78a58012e1bd8f90c874d385dace3228b9a8c80e114ee227fc8a60e7c9611112ceb + checksum: 10c0/dc43617a9398754275e2025af7d5fdee1f2e01b89241fc7881c1206d925e83ad6fe55f439501ae34e734cfbfa5479f6bee3167f4828c913f4f33817d95850189 languageName: node linkType: hard From b00f7d54099c0d26b4bb865dd458662b8d9eb59e Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 30 Sep 2025 17:02:48 +0200 Subject: [PATCH 043/144] refactor: Remote / Publish Connection and constructor --- src/room/InCallView.tsx | 1 - src/state/CallViewModel.ts | 36 ++++++----- src/state/Connection.ts | 112 +++++++++++++++++++-------------- src/state/PublishConnection.ts | 48 ++++++-------- 4 files changed, 104 insertions(+), 93 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index db2c0f2a..57873b40 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -31,7 +31,6 @@ import { VolumeOnSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; -import { ConnectionState } from "livekit-client"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index b6327cfa..6b2ee35a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -122,7 +122,7 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { Connection } from "./Connection"; +import { type Connection, type ConnectionOpts, RemoteConnection } from "./Connection"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; @@ -453,18 +453,21 @@ export class CallViewModel extends ViewModel { private readonly localFocus = makeFocus(this.matrixRTCSession); private readonly localConnection = this.localFocus.then( - (focus) => - new PublishConnection( + (focus) => { + const args: ConnectionOpts = { focus, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, + client: this.matrixRTCSession.room.client, + scope: this.scope, + membershipsFocusMap$: this.membershipsAndFocusMap$, + } + return new PublishConnection( + args, this.mediaDevices, this.muteStates, this.e2eeLivekitOptions(), this.scope.behavior(this.trackProcessorState$), - ), + ) + } ); public readonly livekitConnectionState$ = this.scope.behavior( @@ -521,18 +524,17 @@ export class CallViewModel extends ViewModel { "SFU remoteConnections$ construct new connection: ", focusUrl, ); - nextConnection = new Connection( - { + const args: ConnectionOpts = { + focus: { + type: "livekit", livekit_service_url: focusUrl, livekit_alias: this.livekitAlias, - type: "livekit", }, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, - this.e2eeLivekitOptions(), - ); + client: this.matrixRTCSession.room.client, + scope: this.scope, + membershipsFocusMap$: this.membershipsAndFocusMap$, + } + nextConnection = new RemoteConnection(args, this.e2eeLivekitOptions()); } else { logger.log( "SFU remoteConnections$ use prev connection: ", diff --git a/src/state/Connection.ts b/src/state/Connection.ts index f725ddda..bc352adf 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -7,15 +7,24 @@ Please see LICENSE in the repository root for full details. import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; import { type ConnectionState, type E2EEOptions, Room as LivekitRoom } from "livekit-client"; -import { type MatrixClient } from "matrix-js-sdk"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest } from "rxjs"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; +import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; +export interface ConnectionOpts { + /** The focus server to connect to. */ + focus: LivekitFocus; + /** The Matrix client to use for OpenID and SFU config requests. */ + client: OpenIDClientParts; + /** The observable scope to use for this connection. */ + scope: ObservableScope; + /** An observable of the current RTC call memberships and their associated focus. */ + membershipsFocusMap$: Behavior<{ membership: CallMembership; focus: LivekitFocus }[]>; +} /** * A connection to a Matrix RTC LiveKit backend. * @@ -39,10 +48,20 @@ export class Connection { */ public async start(): Promise { this.stopped = false; - const { url, jwt } = await this.sfuConfig; + // TODO could this be loaded earlier to save time? + const { url, jwt } = await this.getSFUConfigWithOpenID(); + if (!this.stopped) await this.livekitRoom.connect(url, jwt); } + + protected async getSFUConfigWithOpenID(): Promise { + return await getSFUConfigWithOpenID( + this.client, + this.targetFocus.livekit_service_url, + this.targetFocus.livekit_alias + ) + } /** * Stops the connection. * @@ -55,17 +74,6 @@ export class Connection { this.stopped = true; } - protected readonly sfuConfig = getSFUConfigWithOpenID( - this.client, - this.focus.livekit_service_url, - this.focus.livekit_alias - ); - - /* - * An observable of the participants in the livekit room, including subscribers. - * Converts the livekit room events ParticipantConnected/ParticipantDisconnected/StateChange to an observable. - */ - protected readonly participantsIncludingSubscribers$; /** * An observable of the participants that are publishing on this connection. @@ -75,9 +83,9 @@ export class Connection { public readonly publishingParticipants$; /** - * The LiveKit room instance. + * The focus server to connect to. */ - public readonly livekitRoom: LivekitRoom; + protected readonly targetFocus: LivekitFocus; /** * An observable of the livekit connection state. @@ -85,48 +93,39 @@ export class Connection { */ public connectionState$: Behavior; + + private readonly client: OpenIDClientParts; /** * Creates a new connection to a matrix RTC LiveKit backend. * - * @param livekitRoom - Optional LiveKit room instance to use. If not provided, a new instance will be created. - * @param focus - The focus server to connect to. - * @param livekitAlias - The livekit alias to use when connecting to the focus server. TODO duplicate of focus? - * @param client - The matrix client, used to fetch the OpenId token. TODO refactor to avoid passing the whole client - * @param scope - The observable scope to use for creating observables. - * @param membershipsFocusMap$ - The observable of the current call RTC memberships and their associated focus. - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. TODO refactor to avoid passing the whole options? + * @param livekitRoom - LiveKit room instance to use. + * @param opts - Connection options {@link ConnectionOpts}. + * */ - public constructor( - protected readonly focus: LivekitFocus, - // TODO : remove livekitAlias, it's already in focus? - protected readonly livekitAlias: string, - protected readonly client: MatrixClient, - protected readonly scope: ObservableScope, - protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, - e2eeLivekitOptions: E2EEOptions | undefined, - livekitRoom: LivekitRoom | undefined = undefined + protected constructor( + public readonly livekitRoom: LivekitRoom, + opts: ConnectionOpts, ) { - this.livekitRoom = - livekitRoom ?? - new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: e2eeLivekitOptions - }); - this.participantsIncludingSubscribers$ = this.scope.behavior( + const { focus, client, scope, membershipsFocusMap$ } = + opts; + + this.livekitRoom = livekitRoom + this.targetFocus = focus; + this.client = client; + + const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), [] ); - this.publishingParticipants$ = this.scope.behavior( + this.publishingParticipants$ = scope.behavior( combineLatest( - [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], + [participantsIncludingSubscribers$, membershipsFocusMap$], (participants, membershipsFocusMap) => membershipsFocusMap // Find all members that claim to publish on this connection .flatMap(({ membership, focus }) => - focus.livekit_service_url === this.focus.livekit_service_url + focus.livekit_service_url === this.targetFocus.livekit_service_url ? [membership] : [] ) @@ -141,11 +140,32 @@ export class Connection { ), [] ); - this.connectionState$ = this.scope.behavior( + this.connectionState$ = scope.behavior( connectionStateObserver(this.livekitRoom) ); - this.scope.onEnd(() => this.stop()); + scope.onEnd(() => this.stop()); } } +/** + * A remote connection to the Matrix RTC LiveKit backend. + * + * This connection is used for subscribing to remote participants. + * It does not publish any local tracks. + */ +export class RemoteConnection extends Connection { + + /** + * Creates a new remote connection to a matrix RTC LiveKit backend. + * @param opts + * @param sharedE2eeOption - The shared E2EE options to use for the connection. + */ + public constructor(opts: ConnectionOpts, sharedE2eeOption: E2EEOptions | undefined) { + const livekitRoom = new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: sharedE2eeOption + }); + super(livekitRoom, opts); + } +} diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 532be26c..724c6c5f 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -7,9 +7,6 @@ Please see LICENSE in the repository root for full details. import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, Track } from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; -import type { CallMembership, LivekitFocus } from "../../../matrix-js-sdk/lib/matrixrtc"; -import type { MatrixClient } from "../../../matrix-js-sdk"; -import type { ObservableScope } from "./ObservableScope.ts"; import type { Behavior } from "./Behavior.ts"; import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; import type { MuteStates } from "./MuteStates.ts"; @@ -19,7 +16,7 @@ import { getUrlParams } from "../UrlParams.ts"; import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; import { observeTrackReference$ } from "./MediaViewModel.ts"; -import { Connection } from "./Connection.ts"; +import { Connection, type ConnectionOpts } from "./Connection.ts"; /** * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. @@ -39,8 +36,8 @@ export class PublishConnection extends Connection { */ public async start(): Promise { this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); + + await super.start() if (!this.stopped) { // TODO this can throw errors? It will also prompt for permissions if not already granted @@ -60,29 +57,20 @@ export class PublishConnection extends Connection { /** * Creates a new PublishConnection. - * @param focus - The Livekit focus object containing the configuration for the connection. - * @param livekitAlias - TODO: remove, use focus.livekit_alias instead - * @param client - The Matrix client to use for authentication. TODO: remove only pick OpenIDClientParts - * @param scope - The observable scope to use for managing subscriptions. - * @param membershipsFocusMap$ - An observable of the current RTC call memberships and their associated focus. + * @param args - The connection options. {@link ConnectionOpts} * @param devices - The media devices to use for audio and video input. * @param muteStates - The mute states for audio and video. * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). */ public constructor( - focus: LivekitFocus, - livekitAlias: string, - client: MatrixClient, - scope: ObservableScope, - membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, + args: ConnectionOpts, devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, trackerProcessorState$: Behavior ) { + const { scope } = args; logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); @@ -112,17 +100,19 @@ export class PublishConnection extends Connection { }); super( - focus, - livekitAlias, - client, - scope, - membershipsFocusMap$, - e2eeLivekitOptions, - room + room, + args, + // focus, + // livekitAlias, + // client, + // scope, + // membershipsFocusMap$, + // e2eeLivekitOptions, + // room ); // Setup track processor syncing (blur) - const track$ = this.scope.behavior( + const track$ = scope.behavior( observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( map((trackRef) => { const track = trackRef?.publication?.track; @@ -148,7 +138,7 @@ export class PublishConnection extends Connection { } return this.livekitRoom.localParticipant.isCameraEnabled; }); - this.scope.onEnd(() => { + scope.onEnd(() => { this.muteStates.audio.unsetHandler(); this.muteStates.video.unsetHandler(); }); @@ -157,7 +147,7 @@ export class PublishConnection extends Connection { kind: MediaDeviceKind, selected$: Observable ): Subscription => - selected$.pipe(this.scope.bind()).subscribe((device) => { + selected$.pipe(scope.bind()).subscribe((device) => { if (this.connectionState$.value !== ConnectionState.Connected) return; logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", @@ -192,7 +182,7 @@ export class PublishConnection extends Connection { devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - this.scope.bind() + scope.bind() ) .subscribe(() => { if (this.connectionState$.value !== ConnectionState.Connected) return; From 879a1d4af1e96f1ff3b9df5c338f3ee14d881016 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 10:06:43 +0200 Subject: [PATCH 044/144] Connection: add Connection state and handle error on start --- src/state/CallViewModel.ts | 22 ++++++++--- src/state/Connection.ts | 67 ++++++++++++++++++++++++++-------- src/state/PublishConnection.ts | 48 +++++++++++------------- 3 files changed, 89 insertions(+), 48 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6b2ee35a..cac4322e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -470,12 +470,22 @@ export class CallViewModel extends ViewModel { } ); - public readonly livekitConnectionState$ = this.scope.behavior( - combineLatest([this.localConnection]).pipe( - switchMap(([c]) => c.connectionState$), - startWith(ConnectionState.Disconnected), - ), - ); + public readonly livekitConnectionState$ = + this.scope.behavior( + from(this.localConnection).pipe( + switchMap((c) => + c.focusedConnectionState$.pipe( + map((s) => { + if (s.state === "ConnectedToLkRoom") return s.connectionState; + return ConnectionState.Disconnected + }), + distinctUntilChanged(), + ), + ), + startWith(ConnectionState.Disconnected), + ), + ) + /** * The MatrixRTC session participants. diff --git a/src/state/Connection.ts b/src/state/Connection.ts index bc352adf..1e081b06 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -6,9 +6,9 @@ Please see LICENSE in the repository root for full details. */ import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; -import { type ConnectionState, type E2EEOptions, Room as LivekitRoom } from "livekit-client"; +import { type ConnectionState, type E2EEOptions, Room as LivekitRoom, type RoomOptions } from "livekit-client"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest } from "rxjs"; +import { BehaviorSubject, combineLatest } from "rxjs"; import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; @@ -24,7 +24,20 @@ export interface ConnectionOpts { scope: ObservableScope; /** An observable of the current RTC call memberships and their associated focus. */ membershipsFocusMap$: Behavior<{ membership: CallMembership; focus: LivekitFocus }[]>; + + /** Optional factory to create the Livekit room, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; } + +export type FocusConnectionState = + | { state: 'Initialized' } + | { state: 'FetchingConfig', focus: LivekitFocus } + | { state: 'ConnectingToLkRoom', focus: LivekitFocus } + | { state: 'PublishingTracks', focus: LivekitFocus } + | { state: 'FailedToStart', error: Error, focus: LivekitFocus } + | { state: 'ConnectedToLkRoom', connectionState: ConnectionState, focus: LivekitFocus } + | { state: 'Stopped', focus: LivekitFocus }; + /** * A connection to a Matrix RTC LiveKit backend. * @@ -32,6 +45,15 @@ export interface ConnectionOpts { */ export class Connection { + // Private Behavior + private readonly _focusedConnectionState$ = new BehaviorSubject({ state: 'Initialized' }); + + /** + * The current state of the connection to the focus server. + */ + public get focusedConnectionState$(): Behavior { + return this._focusedConnectionState$; + } /** * Whether the connection has been stopped. * @see Connection.stop @@ -48,10 +70,23 @@ export class Connection { */ public async start(): Promise { this.stopped = false; - // TODO could this be loaded earlier to save time? - const { url, jwt } = await this.getSFUConfigWithOpenID(); + try { + this._focusedConnectionState$.next({ state: 'FetchingConfig', focus: this.targetFocus }); + // TODO could this be loaded earlier to save time? + const { url, jwt } = await this.getSFUConfigWithOpenID(); + // If we were stopped while fetching the config, don't proceed to connect + if (this.stopped) return; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); + this._focusedConnectionState$.next({ state: 'ConnectingToLkRoom', focus: this.targetFocus }); + await this.livekitRoom.connect(url, jwt); + // If we were stopped while connecting, don't proceed to update state. + if (this.stopped) return; + + this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', focus: this.targetFocus, connectionState: this.livekitRoom.state }); + } catch (error) { + this._focusedConnectionState$.next({ state: 'FailedToStart', error: error instanceof Error ? error : new Error(`${error}`), focus: this.targetFocus }); + throw error; + } } @@ -71,6 +106,7 @@ export class Connection { public stop(): void { if (this.stopped) return; void this.livekitRoom.disconnect(); + this._focusedConnectionState$.next({ state: 'Stopped', focus: this.targetFocus }); this.stopped = true; } @@ -87,13 +123,6 @@ export class Connection { */ protected readonly targetFocus: LivekitFocus; - /** - * An observable of the livekit connection state. - * Converts the livekit room events StateChange to an observable. - */ - public connectionState$: Behavior; - - private readonly client: OpenIDClientParts; /** * Creates a new connection to a matrix RTC LiveKit backend. @@ -140,9 +169,16 @@ export class Connection { ), [] ); - this.connectionState$ = scope.behavior( + + scope.behavior( connectionStateObserver(this.livekitRoom) - ); + ).subscribe((connectionState) => { + const current = this.focusedConnectionState$.value; + // Only update the state if we are already connected to the LiveKit room. + if (current.state === 'ConnectedToLkRoom') { + this.focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); + } + }); scope.onEnd(() => this.stop()); } @@ -162,7 +198,8 @@ export class RemoteConnection extends Connection { * @param sharedE2eeOption - The shared E2EE options to use for the connection. */ public constructor(opts: ConnectionOpts, sharedE2eeOption: E2EEOptions | undefined) { - const livekitRoom = new LivekitRoom({ + const factory = opts.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const livekitRoom = factory({ ...defaultLiveKitOptions, e2ee: sharedE2eeOption }); diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 724c6c5f..c7b9c6aa 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -4,7 +4,7 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, Track } from "livekit-client"; +import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, type RoomOptions, Track } from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; import type { Behavior } from "./Behavior.ts"; @@ -39,18 +39,20 @@ export class PublishConnection extends Connection { await super.start() - if (!this.stopped) { - // TODO this can throw errors? It will also prompt for permissions if not already granted - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value - }); - for (const track of tracks) { - // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally - // with a timeout. - await this.livekitRoom.localParticipant.publishTrack(track); - // TODO: check if the connection is still active? and break the loop if not? - } + if (this.stopped) return; + + // TODO this can throw errors? It will also prompt for permissions if not already granted + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: this.muteStates.audio.enabled$.value, + video: this.muteStates.video.enabled$.value + }); + if (this.stopped) return; + for (const track of tracks) { + // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally + // with a timeout. + await this.livekitRoom.localParticipant.publishTrack(track); + if (this.stopped) return; + // TODO: check if the connection is still active? and break the loop if not? } }; @@ -74,7 +76,8 @@ export class PublishConnection extends Connection { logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); - const room = new LivekitRoom({ + const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const room = factory({ ...defaultLiveKitOptions, videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, @@ -99,17 +102,7 @@ export class PublishConnection extends Connection { logger.error("Failed to set E2EE enabled on room", e); }); - super( - room, - args, - // focus, - // livekitAlias, - // client, - // scope, - // membershipsFocusMap$, - // e2eeLivekitOptions, - // room - ); + super(room, args); // Setup track processor syncing (blur) const track$ = scope.behavior( @@ -148,7 +141,8 @@ export class PublishConnection extends Connection { selected$: Observable ): Subscription => selected$.pipe(scope.bind()).subscribe((device) => { - if (this.connectionState$.value !== ConnectionState.Connected) return; + if (this.livekitRoom.state != ConnectionState.Connected) return; + // if (this.connectionState$.value !== ConnectionState.Connected) return; logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", this.livekitRoom.getActiveDevice(kind), @@ -185,7 +179,7 @@ export class PublishConnection extends Connection { scope.bind() ) .subscribe(() => { - if (this.connectionState$.value !== ConnectionState.Connected) return; + if (this.livekitRoom.state != ConnectionState.Connected) return; const activeMicTrack = Array.from( this.livekitRoom.localParticipant.audioTrackPublications.values() ).find((d) => d.source === Track.Source.Microphone)?.track; From 3d8639df0331f5d9ffc66766646d5fbf9898713d Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 14:21:37 +0200 Subject: [PATCH 045/144] Connection states tests --- package.json | 1 + src/state/Connection.test.ts | 310 +++++++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 src/state/Connection.test.ts diff --git a/package.json b/package.json index 91583023..ff3d98f6 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "eslint-plugin-react-hooks": "^5.0.0", "eslint-plugin-rxjs": "^5.0.3", "eslint-plugin-unicorn": "^56.0.0", + "fetch-mock": "11.1.5", "global-jsdom": "^26.0.0", "i18next": "^24.0.0", "i18next-browser-languagedetector": "^8.0.0", diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts new file mode 100644 index 00000000..2764a0e1 --- /dev/null +++ b/src/state/Connection.test.ts @@ -0,0 +1,310 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, vi, it, describe, type MockedObject, expect } from "vitest"; +import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; +import { type Room as LivekitRoom } from "livekit-client"; +import fetchMock from "fetch-mock"; + +import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; +import { ObservableScope } from "./ObservableScope.ts"; +import { type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU.ts"; +import { FailToGetOpenIdToken } from "../utils/errors.ts"; + +describe("Start connection states", () => { + + let testScope: ObservableScope; + + let client: MockedObject; + + let fakeLivekitRoom: MockedObject; + + let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; + + const livekitFocus : LivekitFocus = { + livekit_alias:"!roomID:example.org", + livekit_service_url : "https://matrix-rtc.example.org/livekit/jwt" + } + + afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + fetchMock.reset(); + }) + + function setupTest(): void { + testScope = new ObservableScope(); + client = vi.mocked({ + getOpenIdToken: vi.fn().mockResolvedValue( + { + "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", + "token_type": "Bearer", + "matrix_server_name": "example.org", + "expires_in": 3600 + } + ), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + } as unknown as OpenIDClientParts); + fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + + fakeLivekitRoom = vi.mocked({ + connect: vi.fn(), + disconnect: vi.fn(), + remoteParticipants: new Map(), + on: vi.fn(), + off: vi.fn(), + addListener: vi.fn(), + removeListener: vi.fn(), + removeAllListeners: vi.fn(), + } as unknown as LivekitRoom); + + } + + it("start in initialized state", () => { + setupTest(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + const connection = new RemoteConnection( + opts, + undefined, + ); + + expect(connection.focusedConnectionState$.getValue().state) + .toEqual("Initialized"); + }); + + it("fail to getOpenId token then error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState | undefined = undefined; + connection.focusedConnectionState$.subscribe((value) => { + capturedState = value; + }); + + + const deferred = Promise.withResolvers(); + + client.getOpenIdToken.mockImplementation(async () => { + await deferred.promise; + }) + + connection.start() + .catch(() => { + // expected to throw + }) + + expect(capturedState.state).toEqual("FetchingConfig"); + + deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); + + await vi.runAllTimersAsync(); + + if (capturedState.state === "FailedToStart") { + expect(capturedState.error.message).toEqual("Something went wrong"); + expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState.state); + } + + }); + + it("fail to get JWT token and error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState | undefined = undefined; + connection.focusedConnectionState$.subscribe((value) => { + capturedState = value; + }); + + const deferredSFU = Promise.withResolvers(); + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + async () => { + await deferredSFU.promise; + return { + status: 500, + body: "Internal Server Error", + } + } + ); + + + connection.start() + .catch(() => { + // expected to throw + }) + + expect(capturedState.state).toEqual("FetchingConfig"); + + deferredSFU.resolve(); + await vi.runAllTimersAsync(); + + if (capturedState.state === "FailedToStart") { + expect(capturedState.error.message).toContain("SFU Config fetch failed with exception Error"); + expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState.state); + } + + }); + + + it("fail to connect to livekit error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState | undefined = undefined; + connection.focusedConnectionState$.subscribe((value) => { + capturedState = value; + }); + + + const deferredSFU = Promise.withResolvers(); + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockImplementation(async () => { + await deferredSFU.promise; + throw new Error("Failed to connect to livekit"); + }); + + connection.start() + .catch(() => { + // expected to throw + }) + + expect(capturedState.state).toEqual("FetchingConfig"); + + deferredSFU.resolve(); + await vi.runAllTimersAsync(); + + if (capturedState.state === "FailedToStart") { + expect(capturedState.error.message).toContain("Failed to connect to livekit"); + expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState.state); + } + + }); + + it("connection states happy path", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState[] = []; + connection.focusedConnectionState$.subscribe((value) => { + capturedState.push(value); + }); + + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockResolvedValue(undefined); + + await connection.start(); + await vi.runAllTimersAsync(); + + let initialState = capturedState.shift(); + expect(initialState?.state).toEqual("Initialized"); + let fetchingState = capturedState.shift(); + expect(fetchingState?.state).toEqual("FetchingConfig"); + let connectingState = capturedState.shift(); + expect(connectingState?.state).toEqual("ConnectingToLkRoom"); + let connectedState = capturedState.shift(); + expect(connectedState?.state).toEqual("ConnectedToLkRoom"); + + }); + +}) From 47c876f3dfccd0b7bf840799b63869b350afba16 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 14:37:03 +0200 Subject: [PATCH 046/144] lint fixes --- src/state/Connection.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 2764a0e1..59c60c2e 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -270,7 +270,7 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState[] = []; + const capturedState: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -296,13 +296,13 @@ describe("Start connection states", () => { await connection.start(); await vi.runAllTimersAsync(); - let initialState = capturedState.shift(); + const initialState = capturedState.shift(); expect(initialState?.state).toEqual("Initialized"); - let fetchingState = capturedState.shift(); + const fetchingState = capturedState.shift(); expect(fetchingState?.state).toEqual("FetchingConfig"); - let connectingState = capturedState.shift(); + const connectingState = capturedState.shift(); expect(connectingState?.state).toEqual("ConnectingToLkRoom"); - let connectedState = capturedState.shift(); + const connectedState = capturedState.shift(); expect(connectedState?.state).toEqual("ConnectedToLkRoom"); }); From 22900161d62d7b854dbf7f1314ea790b6431738a Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 14:47:45 +0200 Subject: [PATCH 047/144] extract common test setup --- src/state/Connection.test.ts | 68 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 59c60c2e..9fe415c1 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -65,6 +65,42 @@ describe("Start connection states", () => { } + async function setupRemoteConnection(): RemoteConnection { + + setupTest() + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockResolvedValue(undefined); + + const connection = new RemoteConnection( + opts, + undefined, + ); + return connection; + } + it("start in initialized state", () => { setupTest(); @@ -254,45 +290,15 @@ describe("Start connection states", () => { }); it("connection states happy path", async () => { - setupTest(); vi.useFakeTimers(); - const opts: ConnectionOpts = { - client: client, - focus: livekitFocus, - membershipsFocusMap$: fakeMembershipsFocusMap$, - scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } - - const connection = new RemoteConnection( - opts, - undefined, - ); + const connection = setupRemoteConnection(); const capturedState: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedState.push(value); }); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } - } - ); - - fakeLivekitRoom - .connect - .mockResolvedValue(undefined); - await connection.start(); await vi.runAllTimersAsync(); From 6a1f7dd057e4f279570eb91bb1c76521472ea7d4 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 15:23:24 +0200 Subject: [PATCH 048/144] ConnectionState: test livekit connection states --- src/state/Connection.test.ts | 65 +++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 9fe415c1..15c5d88e 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -8,8 +8,9 @@ Please see LICENSE in the repository root for full details. import { afterEach, vi, it, describe, type MockedObject, expect } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { type Room as LivekitRoom } from "livekit-client"; +import { type Room as LivekitRoom, RoomEvent, type RoomEventCallbacks, ConnectionState } from "livekit-client"; import fetchMock from "fetch-mock"; +import EventEmitter from "events"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; @@ -24,6 +25,7 @@ describe("Start connection states", () => { let fakeLivekitRoom: MockedObject; + let fakeRoomEventEmiter: EventEmitter; let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; const livekitFocus : LivekitFocus = { @@ -52,22 +54,23 @@ describe("Start connection states", () => { } as unknown as OpenIDClientParts); fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + fakeRoomEventEmiter = new EventEmitter(); + fakeLivekitRoom = vi.mocked({ connect: vi.fn(), disconnect: vi.fn(), remoteParticipants: new Map(), - on: vi.fn(), - off: vi.fn(), - addListener: vi.fn(), - removeListener: vi.fn(), - removeAllListeners: vi.fn(), + state: ConnectionState.Disconnected, + on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), + off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), + addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), + removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), } as unknown as LivekitRoom); } - async function setupRemoteConnection(): RemoteConnection { - - setupTest() + function setupRemoteConnection(): RemoteConnection { const opts: ConnectionOpts = { client: client, @@ -291,6 +294,7 @@ describe("Start connection states", () => { it("connection states happy path", async () => { vi.useFakeTimers(); + setupTest() const connection = setupRemoteConnection(); @@ -313,4 +317,47 @@ describe("Start connection states", () => { }); + it("should relay livekit events once connected", async () => { + vi.useFakeTimers(); + setupTest() + + const connection = setupRemoteConnection(); + + await connection.start(); + await vi.runAllTimersAsync(); + + const capturedState: FocusConnectionState[] = []; + connection.focusedConnectionState$.subscribe((value) => { + capturedState.push(value); + }); + + const states = [ + ConnectionState.Disconnected, + ConnectionState.Connecting, + ConnectionState.Connected, + ConnectionState.SignalReconnecting, + ConnectionState.Connecting, + ConnectionState.Connected, + ConnectionState.Reconnecting, + ] + for (const state of states) { + fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); + await vi.runAllTimersAsync(); + } + + await vi.runAllTimersAsync(); + + for (const state of states) { + const s = capturedState.shift(); + expect(s?.state).toEqual("ConnectedToLkRoom"); + expect(s?.connectionState).toEqual(state); + + // should always have the focus info + expect(s?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(s?.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); + } + + }); + + }) From e8bf817f881463a6c46c931589ac60b393e92786 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 16:39:21 +0200 Subject: [PATCH 049/144] tests: end scope tests --- src/state/CallViewModel.ts | 2 +- src/state/Connection.test.ts | 65 +++++++++++++++++++++++++++--------- src/state/Connection.ts | 18 ++++++---- 3 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index cac4322e..31a7e32d 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -1890,7 +1890,7 @@ export class CallViewModel extends ViewModel { this.startConnection$ .pipe(this.scope.bind()) .subscribe((c) => void c.start()); - this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => c.stop()); + this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => void c.stop()); combineLatest([this.localFocus, this.join$]) .pipe(this.scope.bind()) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 15c5d88e..8552ec24 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -28,9 +28,9 @@ describe("Start connection states", () => { let fakeRoomEventEmiter: EventEmitter; let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; - const livekitFocus : LivekitFocus = { - livekit_alias:"!roomID:example.org", - livekit_service_url : "https://matrix-rtc.example.org/livekit/jwt" + const livekitFocus: LivekitFocus = { + livekit_alias: "!roomID:example.org", + livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt" } afterEach(() => { @@ -98,8 +98,8 @@ describe("Start connection states", () => { .mockResolvedValue(undefined); const connection = new RemoteConnection( - opts, - undefined, + opts, + undefined, ); return connection; } @@ -115,8 +115,8 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, } const connection = new RemoteConnection( - opts, - undefined, + opts, + undefined, ); expect(connection.focusedConnectionState$.getValue().state) @@ -254,7 +254,7 @@ describe("Start connection states", () => { const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { + () => { return { status: 200, body: @@ -318,15 +318,13 @@ describe("Start connection states", () => { }); it("should relay livekit events once connected", async () => { - vi.useFakeTimers(); setupTest() const connection = setupRemoteConnection(); await connection.start(); - await vi.runAllTimersAsync(); - const capturedState: FocusConnectionState[] = []; + let capturedState: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -342,11 +340,8 @@ describe("Start connection states", () => { ] for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); - await vi.runAllTimersAsync(); } - await vi.runAllTimersAsync(); - for (const state of states) { const s = capturedState.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); @@ -357,7 +352,47 @@ describe("Start connection states", () => { expect(s?.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); } + // If the state is not ConnectedToLkRoom, no events should be relayed anymore + await connection.stop(); + capturedState = []; + for (const state of states) { + fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); + } + + expect(capturedState.length).toEqual(0); + }); -}) + it("shutting down the scope should stop the connection", async () => { + setupTest() + vi.useFakeTimers(); + + const connection = setupRemoteConnection(); + + let capturedState: FocusConnectionState[] = []; + connection.focusedConnectionState$.subscribe((value) => { + capturedState.push(value); + }); + + await connection.start(); + + const stopSpy = vi.spyOn(connection, "stop"); + testScope.end(); + + + expect(stopSpy).toHaveBeenCalled(); + expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); + + /// Ensures that focusedConnectionState$ is bound to the scope. + capturedState = []; + // the subscription should be closed, and no new state should be received + // @ts-expect-error: Accessing private field for testing purposes + connection._focusedConnectionState$.next({ state: "Initialized" }); + // @ts-expect-error: Accessing private field for testing purposes + connection._focusedConnectionState$.next({ state: "ConnectingToLkRoom" }); + + expect(capturedState.length).toEqual(0); + }); + +}); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 1e081b06..1b93b523 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -46,14 +46,14 @@ export type FocusConnectionState = export class Connection { // Private Behavior - private readonly _focusedConnectionState$ = new BehaviorSubject({ state: 'Initialized' }); + private readonly _focusedConnectionState$ + = new BehaviorSubject({ state: 'Initialized' }); /** * The current state of the connection to the focus server. */ - public get focusedConnectionState$(): Behavior { - return this._focusedConnectionState$; - } + public readonly focusedConnectionState$: Behavior; + /** * Whether the connection has been stopped. * @see Connection.stop @@ -103,9 +103,9 @@ export class Connection { * This will disconnect from the LiveKit room. * If the connection is already stopped, this is a no-op. */ - public stop(): void { + public async stop(): Promise { if (this.stopped) return; - void this.livekitRoom.disconnect(); + await this.livekitRoom.disconnect(); this._focusedConnectionState$.next({ state: 'Stopped', focus: this.targetFocus }); this.stopped = true; } @@ -142,6 +142,10 @@ export class Connection { this.targetFocus = focus; this.client = client; + this.focusedConnectionState$ = scope.behavior( + this._focusedConnectionState$, { state: 'Initialized' } + ); + const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), [] @@ -180,7 +184,7 @@ export class Connection { } }); - scope.onEnd(() => this.stop()); + scope.onEnd(() => void this.stop()); } } From dfaa6a33f4fe4c01907d2f405d7aefde7fa58475 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 17:24:19 +0200 Subject: [PATCH 050/144] fix lint errors --- src/state/Connection.test.ts | 215 ++++++++++++++++++----------------- src/state/Connection.ts | 4 +- 2 files changed, 115 insertions(+), 104 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 8552ec24..692aee86 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -5,105 +5,108 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, vi, it, describe, type MockedObject, expect } from "vitest"; +import { afterEach, describe, expect, it, type MockedObject, vi } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { type Room as LivekitRoom, RoomEvent, type RoomEventCallbacks, ConnectionState } from "livekit-client"; +import { ConnectionState, type Room as LivekitRoom, RoomEvent } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; +import { type IOpenIDToken } from "matrix-js-sdk"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; -import { type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU.ts"; +import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../utils/errors.ts"; -describe("Start connection states", () => { - let testScope: ObservableScope; +let testScope: ObservableScope; - let client: MockedObject; +let client: MockedObject; - let fakeLivekitRoom: MockedObject; +let fakeLivekitRoom: MockedObject; - let fakeRoomEventEmiter: EventEmitter; - let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; +let fakeRoomEventEmiter: EventEmitter; +let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; - const livekitFocus: LivekitFocus = { - livekit_alias: "!roomID:example.org", - livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt" +const livekitFocus: LivekitFocus = { + livekit_alias: "!roomID:example.org", + livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", + type: "livekit", +} + +function setupTest(): void { + testScope = new ObservableScope(); + client = vi.mocked({ + getOpenIdToken: vi.fn().mockResolvedValue( + { + "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", + "token_type": "Bearer", + "matrix_server_name": "example.org", + "expires_in": 3600 + } + ), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + } as unknown as OpenIDClientParts); + fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + + fakeRoomEventEmiter = new EventEmitter(); + + fakeLivekitRoom = vi.mocked({ + connect: vi.fn(), + disconnect: vi.fn(), + remoteParticipants: new Map(), + state: ConnectionState.Disconnected, + on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), + off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), + addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), + removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + } as unknown as LivekitRoom); + +} + +function setupRemoteConnection(): RemoteConnection { + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, } + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockResolvedValue(undefined); + + return new RemoteConnection( + opts, + undefined, + ); +} + + +describe("Start connection states", () => { + afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); fetchMock.reset(); }) - function setupTest(): void { - testScope = new ObservableScope(); - client = vi.mocked({ - getOpenIdToken: vi.fn().mockResolvedValue( - { - "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", - "token_type": "Bearer", - "matrix_server_name": "example.org", - "expires_in": 3600 - } - ), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), - } as unknown as OpenIDClientParts); - fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); - - fakeRoomEventEmiter = new EventEmitter(); - - fakeLivekitRoom = vi.mocked({ - connect: vi.fn(), - disconnect: vi.fn(), - remoteParticipants: new Map(), - state: ConnectionState.Disconnected, - on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), - off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), - addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), - removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), - } as unknown as LivekitRoom); - - } - - function setupRemoteConnection(): RemoteConnection { - - const opts: ConnectionOpts = { - client: client, - focus: livekitFocus, - membershipsFocusMap$: fakeMembershipsFocusMap$, - scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } - - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } - } - ); - - fakeLivekitRoom - .connect - .mockResolvedValue(undefined); - - const connection = new RemoteConnection( - opts, - undefined, - ); - return connection; - } - it("start in initialized state", () => { setupTest(); @@ -141,16 +144,16 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState | undefined = undefined; + const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedState = value; + capturedStates.push(value); }); - const deferred = Promise.withResolvers(); + const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation(async () => { - await deferred.promise; + client.getOpenIdToken.mockImplementation(async (): Promise => { + return await deferred.promise; }) connection.start() @@ -158,17 +161,19 @@ describe("Start connection states", () => { // expected to throw }) - expect(capturedState.state).toEqual("FetchingConfig"); + const capturedState = capturedStates.shift(); + expect(capturedState).toBeDefined(); + expect(capturedState!.state).toEqual("FetchingConfig"); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); - if (capturedState.state === "FailedToStart") { - expect(capturedState.error.message).toEqual("Something went wrong"); - expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + if (capturedState!.state === "FailedToStart") { + expect(capturedState!.error.message).toEqual("Something went wrong"); + expect(capturedState!.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); } else { - expect.fail("Expected FailedToStart state but got " + capturedState.state); + expect.fail("Expected FailedToStart state but got " + capturedState?.state); } }); @@ -190,9 +195,9 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState | undefined = undefined; + const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedState = value; + capturedStates.push(value); }); const deferredSFU = Promise.withResolvers(); @@ -213,16 +218,18 @@ describe("Start connection states", () => { // expected to throw }) - expect(capturedState.state).toEqual("FetchingConfig"); + const capturedState = capturedStates.shift(); + expect(capturedState).toBeDefined() + expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); - if (capturedState.state === "FailedToStart") { - expect(capturedState.error.message).toContain("SFU Config fetch failed with exception Error"); - expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + if (capturedState?.state === "FailedToStart") { + expect(capturedState?.error.message).toContain("SFU Config fetch failed with exception Error"); + expect(capturedState?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); } else { - expect.fail("Expected FailedToStart state but got " + capturedState.state); + expect.fail("Expected FailedToStart state but got " + capturedState?.state); } }); @@ -245,9 +252,9 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState | undefined = undefined; + const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedState = value; + capturedStates.push(value) }); @@ -278,16 +285,19 @@ describe("Start connection states", () => { // expected to throw }) - expect(capturedState.state).toEqual("FetchingConfig"); + const capturedState = capturedStates.shift(); + expect(capturedState).toBeDefined() + + expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); - if (capturedState.state === "FailedToStart") { + if (capturedState && capturedState?.state === "FailedToStart") { expect(capturedState.error.message).toContain("Failed to connect to livekit"); expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); } else { - expect.fail("Expected FailedToStart state but got " + capturedState.state); + expect.fail("Expected FailedToStart state but got " + JSON.stringify(capturedState)); } }); @@ -345,11 +355,12 @@ describe("Start connection states", () => { for (const state of states) { const s = capturedState.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); - expect(s?.connectionState).toEqual(state); + const connectedState = s as FocusConnectionState & { state: "ConnectedToLkRoom" }; + expect(connectedState.connectionState).toEqual(state); // should always have the focus info - expect(s?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); - expect(s?.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); + expect(connectedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(connectedState.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); } // If the state is not ConnectedToLkRoom, no events should be relayed anymore diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 1b93b523..16dd2607 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -177,10 +177,10 @@ export class Connection { scope.behavior( connectionStateObserver(this.livekitRoom) ).subscribe((connectionState) => { - const current = this.focusedConnectionState$.value; + const current = this._focusedConnectionState$.value; // Only update the state if we are already connected to the LiveKit room. if (current.state === 'ConnectedToLkRoom') { - this.focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); + this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); } }); From 68aae4a8e3e2cd208db5ccc402b134b34159af5c Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 2 Oct 2025 11:23:11 +0200 Subject: [PATCH 051/144] fix another rename + another js-sdk bump Signed-off-by: Timo K --- src/utils/test.ts | 10 +++++++--- yarn.lock | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/utils/test.ts b/src/utils/test.ts index 842ca008..519fdd50 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -16,12 +16,13 @@ import { } from "matrix-js-sdk"; import { CallMembership, - type Focus, + type Transport, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, MembershipManagerEvent, type SessionMembershipData, Status, + type LivekitFocusSelection, } from "matrix-js-sdk/lib/matrixrtc"; import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { @@ -172,8 +173,11 @@ export function mockRtcMembership( user: string | RoomMember, deviceId: string, callId = "", - fociPreferred: Focus[] = [], - focusActive: Focus = { type: "oldest_membership" }, + fociPreferred: Transport[] = [], + focusActive: LivekitFocusSelection = { + type: "livekit", + focus_selection: "oldest_membership", + }, membership: Partial = {}, ): CallMembership { const data: SessionMembershipData = { diff --git a/yarn.lock b/yarn.lock index a149eaf5..197cee3e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10299,7 +10299,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=d94d02d19b9f17c724b5919b185fea3413dbf7a2" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=a343e8c92a5a37f419eb1b762db3a123e41ef66d" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10315,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/dc43617a9398754275e2025af7d5fdee1f2e01b89241fc7881c1206d925e83ad6fe55f439501ae34e734cfbfa5479f6bee3167f4828c913f4f33817d95850189 + checksum: 10c0/4893878f2fe07b06334bab4674a01569037d0f3e737fef3f0bb97a98b01d71fc304627921673f128821a17d824de9b63cc06456db15f9d45eb10bba1ceacd5c5 languageName: node linkType: hard From 0502f66e21ac31ce07e5511f96aca3fab676c7fa Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 2 Oct 2025 12:53:59 +0200 Subject: [PATCH 052/144] tests: Add publisher observable tests --- src/state/Connection.test.ts | 203 +++++++++++++++++++++++++++-------- 1 file changed, 159 insertions(+), 44 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 692aee86..5c725e83 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, type MockedObject, vi } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { ConnectionState, type Room as LivekitRoom, RoomEvent } from "livekit-client"; +import { ConnectionState, type RemoteParticipant, type Room as LivekitRoom, RoomEvent } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; @@ -31,8 +31,8 @@ let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focu const livekitFocus: LivekitFocus = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", - type: "livekit", -} + type: "livekit" +}; function setupTest(): void { testScope = new ObservableScope(); @@ -45,7 +45,7 @@ function setupTest(): void { "expires_in": 3600 } ), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + getDeviceId: vi.fn().mockReturnValue("ABCDEF") } as unknown as OpenIDClientParts); fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); @@ -60,7 +60,7 @@ function setupTest(): void { off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter) } as unknown as LivekitRoom); } @@ -72,8 +72,8 @@ function setupRemoteConnection(): RemoteConnection { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { @@ -82,9 +82,9 @@ function setupRemoteConnection(): RemoteConnection { body: { "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } + "jwt": "ATOKEN" + } + }; } ); @@ -94,7 +94,7 @@ function setupRemoteConnection(): RemoteConnection { return new RemoteConnection( opts, - undefined, + undefined ); } @@ -105,7 +105,7 @@ describe("Start connection states", () => { vi.useRealTimers(); vi.clearAllMocks(); fetchMock.reset(); - }) + }); it("start in initialized state", () => { setupTest(); @@ -115,11 +115,11 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); expect(connection.focusedConnectionState$.getValue().state) @@ -135,13 +135,13 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); const capturedStates: FocusConnectionState[] = []; @@ -154,14 +154,14 @@ describe("Start connection states", () => { client.getOpenIdToken.mockImplementation(async (): Promise => { return await deferred.promise; - }) + }); connection.start() .catch(() => { // expected to throw - }) + }); - const capturedState = capturedStates.shift(); + let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); expect(capturedState!.state).toEqual("FetchingConfig"); @@ -169,6 +169,7 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); + capturedState = capturedStates.pop(); if (capturedState!.state === "FailedToStart") { expect(capturedState!.error.message).toEqual("Something went wrong"); expect(capturedState!.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); @@ -187,12 +188,12 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); const capturedStates: FocusConnectionState[] = []; @@ -207,8 +208,8 @@ describe("Start connection states", () => { await deferredSFU.promise; return { status: 500, - body: "Internal Server Error", - } + body: "Internal Server Error" + }; } ); @@ -216,15 +217,17 @@ describe("Start connection states", () => { connection.start() .catch(() => { // expected to throw - }) + }); - const capturedState = capturedStates.shift(); - expect(capturedState).toBeDefined() + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); + capturedState = capturedStates.pop(); + if (capturedState?.state === "FailedToStart") { expect(capturedState?.error.message).toContain("SFU Config fetch failed with exception Error"); expect(capturedState?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); @@ -244,17 +247,17 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedStates.push(value) + capturedStates.push(value); }); @@ -267,9 +270,9 @@ describe("Start connection states", () => { body: { "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } + "jwt": "ATOKEN" + } + }; } ); @@ -283,16 +286,18 @@ describe("Start connection states", () => { connection.start() .catch(() => { // expected to throw - }) + }); - const capturedState = capturedStates.shift(); - expect(capturedState).toBeDefined() + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); + capturedState = capturedStates.pop(); + if (capturedState && capturedState?.state === "FailedToStart") { expect(capturedState.error.message).toContain("Failed to connect to livekit"); expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); @@ -304,7 +309,7 @@ describe("Start connection states", () => { it("connection states happy path", async () => { vi.useFakeTimers(); - setupTest() + setupTest(); const connection = setupRemoteConnection(); @@ -328,7 +333,7 @@ describe("Start connection states", () => { }); it("should relay livekit events once connected", async () => { - setupTest() + setupTest(); const connection = setupRemoteConnection(); @@ -346,8 +351,8 @@ describe("Start connection states", () => { ConnectionState.SignalReconnecting, ConnectionState.Connecting, ConnectionState.Connected, - ConnectionState.Reconnecting, - ] + ConnectionState.Reconnecting + ]; for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); } @@ -376,7 +381,7 @@ describe("Start connection states", () => { it("shutting down the scope should stop the connection", async () => { - setupTest() + setupTest(); vi.useFakeTimers(); const connection = setupRemoteConnection(); @@ -407,3 +412,113 @@ describe("Start connection states", () => { }); }); + + +function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { + return vi.mocked({ + identity: id + } as unknown as RemoteParticipant); +} + +function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { + return vi.mocked({ + sender: userId, + deviceId: deviceId, + } as unknown as CallMembership); +} + +describe("Publishing participants observations", () => { + + + it("should emit the list of publishing participants", async () => { + setupTest(); + + const connection = setupRemoteConnection(); + + const bobIsAPublisher = Promise.withResolvers(); + const danIsAPublisher = Promise.withResolvers(); + const observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + connection.publishingParticipants$.subscribe((publishers) => { + observedPublishers.push(publishers); + if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } + }); + // The publishingParticipants$ observable is derived from the current members of the + // livekitRoom and the rtc membership in order to publish the members that are publishing + // on this connection. + + let participants: RemoteParticipant[]= [ + fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), + fakeRemoteLivekitParticipant("@dan:example.org:DEV333") + ]; + + // Let's simulate 3 members on the livekitRoom + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + + for (const participant of participants) { + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + } + + // At this point there should be no publishers + expect(observedPublishers.pop()!.length).toEqual(0); + + + const otherFocus: LivekitFocus = { + livekit_alias: "!roomID:example.org", + livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", + type: "livekit" + } + + + const rtcMemberships = [ + // Say bob is on the same focus + { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, + // Alice and carol is on a different focus + { membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), focus: otherFocus }, + { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), focus: otherFocus }, + // NO DAVE YET + ]; + // signal this change in rtc memberships + fakeMembershipsFocusMap$.next(rtcMemberships); + + // We should have bob has a publisher now + await bobIsAPublisher.promise; + const publishers = observedPublishers.pop(); + expect(publishers?.length).toEqual(1); + expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + + // Now let's make dan join the rtc memberships + rtcMemberships + .push({ membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), focus: livekitFocus }); + fakeMembershipsFocusMap$.next(rtcMemberships); + + // We should have bob and dan has publishers now + await danIsAPublisher.promise; + const twoPublishers = observedPublishers.pop(); + expect(twoPublishers?.length).toEqual(2); + expect(twoPublishers?.some((p) => p.participant.identity === "@bob:example.org:DEV111")).toBeTruthy(); + expect(twoPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + + // Now let's make bob leave the livekit room + participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + + const updatedPublishers = observedPublishers.pop(); + expect(updatedPublishers?.length).toEqual(1); + expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + }) + +}); From 84f95be48d8715eefa48b85deb89ada4f1da0889 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 2 Oct 2025 13:08:00 +0200 Subject: [PATCH 053/144] test: Ensure scope for publishers observer --- src/state/Connection.test.ts | 69 ++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 5c725e83..5529e588 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -99,13 +99,14 @@ function setupRemoteConnection(): RemoteConnection { } -describe("Start connection states", () => { +afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + fetchMock.reset(); +}); - afterEach(() => { - vi.useRealTimers(); - vi.clearAllMocks(); - fetchMock.reset(); - }); + +describe("Start connection states", () => { it("start in initialized state", () => { setupTest(); @@ -521,4 +522,60 @@ describe("Publishing participants observations", () => { expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); }) + + it("should be scoped to parent scope", async () => { + setupTest(); + + const connection = setupRemoteConnection(); + + let observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + connection.publishingParticipants$.subscribe((publishers) => { + observedPublishers.push(publishers); + }); + + let participants: RemoteParticipant[]= [ + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + ]; + + // Let's simulate 3 members on the livekitRoom + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + + for (const participant of participants) { + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + } + + // At this point there should be no publishers + expect(observedPublishers.pop()!.length).toEqual(0); + + const rtcMemberships = [ + // Say bob is on the same focus + { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, + ]; + // signal this change in rtc memberships + fakeMembershipsFocusMap$.next(rtcMemberships); + + // We should have bob has a publisher now + const publishers = observedPublishers.pop(); + expect(publishers?.length).toEqual(1); + expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + + // end the parent scope + testScope.end(); + observedPublishers = []; + + // SHOULD NOT emit any more publishers as the scope is ended + participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + + expect(observedPublishers.length).toEqual(0); + }) + + }); From 00401ca38ab99aa0fd32469425768ffc7d99ffa5 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 2 Oct 2025 15:15:23 +0200 Subject: [PATCH 054/144] refactor: PublishConnection extract from giant constructor --- src/state/PublishConnection.ts | 278 +++++++++++++++++++-------------- 1 file changed, 162 insertions(+), 116 deletions(-) diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index c7b9c6aa..6c15fc0f 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -4,7 +4,14 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, type RoomOptions, Track } from "livekit-client"; +import { + ConnectionState, + type E2EEOptions, + LocalVideoTrack, + Room as LivekitRoom, + type RoomOptions, + Track +} from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; import type { Behavior } from "./Behavior.ts"; @@ -17,6 +24,7 @@ import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; import { observeTrackReference$ } from "./MediaViewModel.ts"; import { Connection, type ConnectionOpts } from "./Connection.ts"; +import { type ObservableScope } from "./ObservableScope.ts"; /** * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. @@ -24,6 +32,44 @@ import { Connection, type ConnectionOpts } from "./Connection.ts"; */ export class PublishConnection extends Connection { + /** + * Creates a new PublishConnection. + * @param args - The connection options. {@link ConnectionOpts} + * @param devices - The media devices to use for audio and video input. + * @param muteStates - The mute states for audio and video. + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. + * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). + */ + public constructor( + args: ConnectionOpts, + devices: MediaDevices, + private readonly muteStates: MuteStates, + e2eeLivekitOptions: E2EEOptions | undefined, + trackerProcessorState$: Behavior + ) { + const { scope } = args; + logger.info("[LivekitRoom] Create LiveKit room"); + const { controlledAudioDevices } = getUrlParams(); + + const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const room = factory( + generateRoomOption(devices, trackerProcessorState$.value, controlledAudioDevices, e2eeLivekitOptions) + ); + room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + + super(room, args); + + // Setup track processor syncing (blur) + this.observeTrackProcessors(scope, room, trackerProcessorState$); + // Observe mute state changes and update LiveKit microphone/camera states accordingly + this.observeMuteStates(scope); + // Observe media device changes and update LiveKit active devices accordingly + this.observeMediaDevices(scope, devices, controlledAudioDevices); + + this.workaroundRestartAudioInputTrackChrome(devices, scope); + } /** * Start the connection to LiveKit and publish local tracks. @@ -56,123 +102,18 @@ export class PublishConnection extends Connection { } }; + /// Private methods - /** - * Creates a new PublishConnection. - * @param args - The connection options. {@link ConnectionOpts} - * @param devices - The media devices to use for audio and video input. - * @param muteStates - The mute states for audio and video. - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. - * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). - */ - public constructor( - args: ConnectionOpts, - devices: MediaDevices, - private readonly muteStates: MuteStates, - e2eeLivekitOptions: E2EEOptions | undefined, - trackerProcessorState$: Behavior - ) { - const { scope } = args; - logger.info("[LivekitRoom] Create LiveKit room"); - const { controlledAudioDevices } = getUrlParams(); + // Restart the audio input track whenever we detect that the active media + // device has changed to refer to a different hardware device. We do this + // for the sake of Chrome, which provides a "default" device that is meant + // to match the system's default audio input, whatever that may be. + // This is special-cased for only audio inputs because we need to dig around + // in the LocalParticipant object for the track object and there's not a nice + // way to do that generically. There is usually no OS-level default video capture + // device anyway, and audio outputs work differently. + private workaroundRestartAudioInputTrackChrome(devices: MediaDevices, scope: ObservableScope): void { - const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); - const room = factory({ - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: trackerProcessorState$.value.processor - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id - }, - e2ee: e2eeLivekitOptions - }); - room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - super(room, args); - - // Setup track processor syncing (blur) - const track$ = scope.behavior( - observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }) - ) - ); - trackProcessorSync(track$, trackerProcessorState$); - - this.muteStates.audio.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); - } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; - }); - this.muteStates.video.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); - } - return this.livekitRoom.localParticipant.isCameraEnabled; - }); - scope.onEnd(() => { - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - }); - - const syncDevice = ( - kind: MediaDeviceKind, - selected$: Observable - ): Subscription => - selected$.pipe(scope.bind()).subscribe((device) => { - if (this.livekitRoom.state != ConnectionState.Connected) return; - // if (this.connectionState$.value !== ConnectionState.Connected) return; - logger.info( - "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - this.livekitRoom.getActiveDevice(kind), - " !== ", - device?.id - ); - if ( - device !== undefined && - this.livekitRoom.getActiveDevice(kind) !== device.id - ) { - this.livekitRoom - .switchActiveDevice(kind, device.id) - .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e) - ); - } - }); - - syncDevice("audioinput", devices.audioInput.selected$); - if (!controlledAudioDevices) - syncDevice("audiooutput", devices.audioOutput.selected$); - syncDevice("videoinput", devices.videoInput.selected$); - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), @@ -205,4 +146,109 @@ export class PublishConnection extends Connection { } }); } + +// Observe changes in the selected media devices and update the LiveKit room accordingly. + private observeMediaDevices(scope: ObservableScope, devices: MediaDevices, controlledAudioDevices: boolean):void { + const syncDevice = ( + kind: MediaDeviceKind, + selected$: Observable + ): Subscription => + selected$.pipe(scope.bind()).subscribe((device) => { + if (this.livekitRoom.state != ConnectionState.Connected) return; + // if (this.connectionState$.value !== ConnectionState.Connected) return; + logger.info( + "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", + this.livekitRoom.getActiveDevice(kind), + " !== ", + device?.id + ); + if ( + device !== undefined && + this.livekitRoom.getActiveDevice(kind) !== device.id + ) { + this.livekitRoom + .switchActiveDevice(kind, device.id) + .catch((e) => + logger.error(`Failed to sync ${kind} device with LiveKit`, e) + ); + } + }); + + syncDevice("audioinput", devices.audioInput.selected$); + if (!controlledAudioDevices) + syncDevice("audiooutput", devices.audioOutput.selected$); + syncDevice("videoinput", devices.videoInput.selected$); + } + + /** + * Observe changes in the mute states and update the LiveKit room accordingly. + * @param scope + * @private + */ + private observeMuteStates(scope: ObservableScope): void { + this.muteStates.audio.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit audio input mute state", e); + } + return this.livekitRoom.localParticipant.isMicrophoneEnabled; + }); + this.muteStates.video.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setCameraEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit video input mute state", e); + } + return this.livekitRoom.localParticipant.isCameraEnabled; + }); + scope.onEnd(() => { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + }); + } + + private observeTrackProcessors(scope: ObservableScope, room: LivekitRoom, trackerProcessorState$: Behavior): void { + const track$ = scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( + map((trackRef) => { + const track = trackRef?.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }) + ) + ); + trackProcessorSync(track$, trackerProcessorState$); + } + +} + + +// Generate the initial LiveKit RoomOptions based on the current media devices and processor state. +function generateRoomOption( + devices: MediaDevices, + processorState: ProcessorState, + controlledAudioDevices: boolean, + e2eeLivekitOptions: E2EEOptions | undefined, +): RoomOptions { + return { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: processorState.processor + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id + }, + e2ee: e2eeLivekitOptions + }; } From 86fb026be86d7ea0fa904babd920b6e1d0b9f1c0 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Oct 2025 14:43:22 -0400 Subject: [PATCH 055/144] Turn multi-SFU media transport into a developer option --- locales/en/app.json | 3 +- src/room/GroupCallErrorBoundary.test.tsx | 8 +- src/room/GroupCallView.test.tsx | 4 +- src/rtcSessionHelpers.ts | 64 ++- src/settings/DeveloperSettingsTab.tsx | 16 + src/settings/settings.ts | 2 + src/state/Async.ts | 44 ++ src/state/CallViewModel.ts | 522 ++++++++++++++--------- src/state/Connection.ts | 57 +-- src/state/MuteStates.ts | 2 +- src/state/ObservableScope.ts | 19 +- src/utils/errors.ts | 10 +- 12 files changed, 461 insertions(+), 290 deletions(-) create mode 100644 src/state/Async.ts diff --git a/locales/en/app.json b/locales/en/app.json index dc027c92..704f68ac 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -72,6 +72,7 @@ "livekit_server_info": "LiveKit Server Info", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", + "multi_sfu": "Multi-SFU media transport", "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", "url_params": "URL parameters", @@ -91,7 +92,7 @@ "generic_description": "Submitting debug logs will help us track down the problem.", "insufficient_capacity": "Insufficient capacity", "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.", - "matrix_rtc_focus_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", + "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/src/room/GroupCallErrorBoundary.test.tsx b/src/room/GroupCallErrorBoundary.test.tsx index 51912956..22338924 100644 --- a/src/room/GroupCallErrorBoundary.test.tsx +++ b/src/room/GroupCallErrorBoundary.test.tsx @@ -26,7 +26,7 @@ import { E2EENotSupportedError, type ElementCallError, InsufficientCapacityError, - MatrixRTCFocusMissingError, + MatrixRTCTransportMissingError, UnknownCallError, } from "../utils/errors.ts"; import { mockConfig } from "../utils/test.ts"; @@ -34,7 +34,7 @@ import { ElementWidgetActions, type WidgetHelpers } from "../widget.ts"; test.each([ { - error: new MatrixRTCFocusMissingError("example.com"), + error: new MatrixRTCTransportMissingError("example.com"), expectedTitle: "Call is not supported", }, { @@ -85,7 +85,7 @@ test.each([ ); test("should render the error page with link back to home", async () => { - const error = new MatrixRTCFocusMissingError("example.com"); + const error = new MatrixRTCTransportMissingError("example.com"); const TestComponent = (): ReactNode => { throw error; }; @@ -213,7 +213,7 @@ describe("Rageshake button", () => { }); test("should have a close button in widget mode", async () => { - const error = new MatrixRTCFocusMissingError("example.com"); + const error = new MatrixRTCTransportMissingError("example.com"); const TestComponent = (): ReactNode => { throw error; }; diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index bf5d1fef..b8bc2f53 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -42,7 +42,7 @@ import { import { GroupCallView } from "./GroupCallView"; import { type WidgetHelpers } from "../widget"; import { LazyEventEmitter } from "../LazyEventEmitter"; -import { MatrixRTCFocusMissingError } from "../utils/errors"; +import { MatrixRTCTransportMissingError } from "../utils/errors"; import { ProcessorProvider } from "../livekit/TrackProcessorContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; @@ -258,7 +258,7 @@ test("GroupCallView leaves the session when an error occurs", async () => { test("GroupCallView shows errors that occur during joining", async () => { const user = userEvent.setup(); - enterRTCSession.mockRejectedValue(new MatrixRTCFocusMissingError("")); + enterRTCSession.mockRejectedValue(new MatrixRTCTransportMissingError("")); onTestFinished(() => { enterRTCSession.mockReset(); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 175b35f4..3cdd82e7 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -17,7 +17,7 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; import { Config } from "./config/Config"; import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; -import { MatrixRTCFocusMissingError } from "./utils/errors"; +import { MatrixRTCTransportMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; @@ -28,35 +28,31 @@ export function getLivekitAlias(rtcSession: MatrixRTCSession): string { return rtcSession.room.roomId; } -async function makeFocusInternal( +async function makeTransportInternal( rtcSession: MatrixRTCSession, ): Promise { - logger.log("Searching for a preferred focus"); + logger.log("Searching for a preferred transport"); const livekitAlias = getLivekitAlias(rtcSession); - const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth"); + // TODO-MULTI-SFU: Either remove this dev tool or make it more official + const urlFromStorage = + localStorage.getItem("robin-matrixrtc-auth") ?? + localStorage.getItem("timo-focus-url"); if (urlFromStorage !== null) { - const focusFromStorage: LivekitTransport = { + const transportFromStorage: LivekitTransport = { type: "livekit", livekit_service_url: urlFromStorage, livekit_alias: livekitAlias, }; - logger.log("Using LiveKit focus from local storage: ", focusFromStorage); - return focusFromStorage; + logger.log( + "Using LiveKit transport from local storage: ", + transportFromStorage, + ); + return transportFromStorage; } // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); - if (localStorage.getItem("timo-focus-url")) { - const timoFocusUrl = localStorage.getItem("timo-focus-url")!; - const focusFromUrl: LivekitTransport = { - type: "livekit", - livekit_service_url: timoFocusUrl, - livekit_alias: livekitAlias, - }; - logger.log("Using LiveKit focus from localStorage: ", timoFocusUrl); - return focusFromUrl; - } if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started @@ -64,46 +60,46 @@ async function makeFocusInternal( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - const focus: LivekitTransportConfig | undefined = wellKnownFoci.find( + const transport: LivekitTransportConfig | undefined = wellKnownFoci.find( (f) => f && isLivekitTransportConfig(f), ); - if (focus !== undefined) { - logger.log("Using LiveKit focus from .well-known: ", focus); - return { ...focus, livekit_alias: livekitAlias }; + if (transport !== undefined) { + logger.log("Using LiveKit transport from .well-known: ", transport); + return { ...transport, livekit_alias: livekitAlias }; } } } const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { - const focusFromConf: LivekitTransport = { + const transportFromConf: LivekitTransport = { type: "livekit", livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - logger.log("Using LiveKit focus from config: ", focusFromConf); - return focusFromConf; + logger.log("Using LiveKit transport from config: ", transportFromConf); + return transportFromConf; } - throw new MatrixRTCFocusMissingError(domain ?? ""); + throw new MatrixRTCTransportMissingError(domain ?? ""); } -export async function makeFocus( +export async function makeTransport( rtcSession: MatrixRTCSession, ): Promise { - const focus = await makeFocusInternal(rtcSession); + const transport = await makeTransportInternal(rtcSession); // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID( rtcSession.room.client, - focus.livekit_service_url, - focus.livekit_alias, + transport.livekit_service_url, + transport.livekit_alias, ); - return focus; + return transport; } export async function enterRTCSession( rtcSession: MatrixRTCSession, - focus: LivekitTransport, + transport: LivekitTransport, encryptMedia: boolean, useNewMembershipManager = true, useExperimentalToDeviceTransport = false, @@ -120,10 +116,10 @@ export async function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - // Multi-sfu does not need a focus preferred list. just the focus that is actually used. + // Multi-sfu does not need a preferred foci list. just the focus that is actually used. rtcSession.joinRoomSession( - useMultiSfu ? [focus] : [], - useMultiSfu ? focus : undefined, + useMultiSfu ? [] : [transport], + useMultiSfu ? transport : undefined, { notificationType, callIntent, diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 1949ecf7..36c8a2e6 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -16,6 +16,7 @@ import { showConnectionStats as showConnectionStatsSetting, useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, + multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, } from "./settings"; @@ -50,6 +51,7 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { useExperimentalToDeviceTransport, setUseExperimentalToDeviceTransport, ] = useSetting(useExperimentalToDeviceTransportSetting); + const [multiSfu, setMultiSfu] = useSetting(multiSfuSetting); const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); @@ -166,6 +168,20 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { )} /> + + ): void => { + setMultiSfu(event.target.checked); + }, + [setMultiSfu], + )} + /> + ( true, ); +export const multiSfu = new Setting("multi-sfu", false); + export const muteAllAudio = new Setting("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); diff --git a/src/state/Async.ts b/src/state/Async.ts new file mode 100644 index 00000000..2baa674c --- /dev/null +++ b/src/state/Async.ts @@ -0,0 +1,44 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + catchError, + from, + map, + Observable, + of, + startWith, + switchMap, +} from "rxjs"; + +export type Async = + | { state: "loading" } + | { state: "error"; value: Error } + | { state: "ready"; value: A }; + +export const loading: Async = { state: "loading" }; +export function error(value: Error): Async { + return { state: "error", value }; +} +export function ready(value: A): Async { + return { state: "ready", value }; +} + +export function async(promise: Promise): Observable> { + return from(promise).pipe( + map(ready), + startWith(loading), + catchError((e) => of(error(e))), + ); +} + +export function mapAsync( + async: Async, + project: (value: A) => B, +): Async { + return async.state === "ready" ? ready(project(async.value)) : async; +} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 2f4bfa0c..8988e518 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,6 +28,7 @@ import { EventType, RoomEvent, } from "matrix-js-sdk"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; import { BehaviorSubject, EMPTY, @@ -48,6 +49,7 @@ import { of, pairwise, race, + repeat, scan, skip, skipWhile, @@ -57,6 +59,7 @@ import { switchScan, take, takeUntil, + takeWhile, tap, throttleTime, timer, @@ -65,6 +68,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitTransport, + type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -90,6 +94,7 @@ import { import { ObservableScope } from "./ObservableScope"; import { duplicateTiles, + multiSfu, playReactionsSound, showReactions, } from "../settings/settings"; @@ -118,7 +123,7 @@ import { constant, type Behavior } from "./Behavior"; import { enterRTCSession, getLivekitAlias, - makeFocus, + makeTransport, } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; @@ -127,6 +132,7 @@ import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; +import { type Async, async, mapAsync, ready } from "./Async"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -449,27 +455,33 @@ export class CallViewModel extends ViewModel { } : undefined; - private readonly localFocus = makeFocus(this.matrixRTCSession); + private readonly join$ = new Subject(); - private readonly localConnection = this.localFocus.then( - (focus) => - new PublishConnection( - focus, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - ); + public join(): void { + this.join$.next(); + } - public readonly livekitConnectionState$ = this.scope.behavior( - combineLatest([this.localConnection]).pipe( - switchMap(([c]) => c.connectionState$), - startWith(ConnectionState.Disconnected), + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + private readonly leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether we are joined to the call. This reflects our local state rather + * than whether all connections are truly up and running. + */ + private readonly joined$ = this.scope.behavior( + this.join$.pipe( + map(() => true), + // Using takeUntil with the repeat operator is perfectly valid. + // eslint-disable-next-line rxjs/no-unsafe-takeuntil + takeUntil(this.leaveHoisted$), + endWith(false), + repeat(), + startWith(false), ), ); @@ -488,125 +500,224 @@ export class CallViewModel extends ViewModel { ), ); - private readonly membershipsAndFocusMap$ = this.scope.behavior( - this.memberships$.pipe( - map((memberships) => - memberships.flatMap((m) => { - const f = this.matrixRTCSession.resolveActiveFocus(m); - return f && isLivekitTransport(f) - ? [{ membership: m, focus: f }] - : []; - }), + /** + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). + */ + private readonly preferredTransport = makeTransport(this.matrixRTCSession); + + /** + * Lists the transports used by ourselves, plus all other MatrixRTC session + * members. + */ + private readonly transports$: Behavior<{ + local: Async; + remote: { membership: CallMembership; transport: LivekitTransport }[]; + } | null> = this.scope.behavior( + this.joined$.pipe( + switchMap((joined) => + joined + ? combineLatest( + [ + async(this.preferredTransport), + this.memberships$, + multiSfu.value$, + ], + (preferred, memberships, multiSfu) => { + const remote = memberships.flatMap((m) => { + if (m.sender === this.userId && m.deviceId === this.deviceId) + return []; + const t = this.matrixRTCSession.resolveActiveFocus(m); + return t && isLivekitTransport(t) + ? [{ membership: m, transport: t }] + : []; + }); + let local = preferred; + if (!multiSfu) { + const oldest = this.matrixRTCSession.getOldestMembership(); + if (oldest !== undefined) { + const selection = oldest.getTransport(oldest); + if (isLivekitTransport(selection)) local = ready(selection); + } + } + return { local, remote }; + }, + ) + : of(null), ), ), ); - private readonly livekitServiceUrls$ = this.membershipsAndFocusMap$.pipe( - map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))), + /** + * Lists the transports used by each MatrixRTC session member other than + * ourselves. + */ + private readonly remoteTransports$ = this.scope.behavior( + this.transports$.pipe(map((transports) => transports?.remote ?? [])), ); + /** + * The transport over which we should be actively publishing our media. + */ + private readonly localTransport$: Behavior | null> = + this.scope.behavior( + this.transports$.pipe( + map((transports) => transports?.local ?? null), + distinctUntilChanged(deepCompare), + ), + ); + + private readonly localConnectionAndTransport$ = this.scope.behavior( + this.localTransport$.pipe( + map( + (transport) => + transport && + mapAsync(transport, (transport) => ({ + connection: new PublishConnection( + transport, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + this.remoteTransports$, + this.mediaDevices, + this.muteStates, + this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), + ), + transport, + })), + ), + ), + ); + + private readonly localConnection$ = this.scope.behavior( + this.localConnectionAndTransport$.pipe( + map((value) => value && mapAsync(value, ({ connection }) => connection)), + ), + ); + + public readonly livekitConnectionState$ = this.scope.behavior( + this.localConnection$.pipe( + switchMap((c) => + c?.state === "ready" + ? c.value.connectionState$ + : of(ConnectionState.Disconnected), + ), + ), + ); + + /** + * Connections for each transport in use by one or more session members that + * is *distinct* from the local transport. + */ private readonly remoteConnections$ = this.scope.behavior( - combineLatest([this.localFocus, this.livekitServiceUrls$]).pipe( - accumulate( - new Map(), - (prev, [localFocus, focusUrls]) => { - const stopped = new Map(prev); - const next = new Map(); - for (const focusUrl of focusUrls) { - if (focusUrl !== localFocus.livekit_service_url) { - stopped.delete(focusUrl); + this.transports$.pipe( + accumulate(new Map(), (prev, transports) => { + const next = new Map(); - let nextConnection = prev.get(focusUrl); - if (!nextConnection) { - logger.log( - "SFU remoteConnections$ construct new connection: ", - focusUrl, - ); - nextConnection = new Connection( - { - livekit_service_url: focusUrl, - livekit_alias: this.livekitAlias, - type: "livekit", - }, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, - this.e2eeLivekitOptions(), - ); - } else { - logger.log( - "SFU remoteConnections$ use prev connection: ", - focusUrl, - ); - } - next.set(focusUrl, nextConnection); + // Until the local transport becomes ready we have no idea which + // transports will actually need a dedicated remote connection + if (transports?.local.state === "ready") { + const localServiceUrl = transports.local.value.livekit_service_url; + const remoteServiceUrls = new Set( + transports.remote.flatMap(({ membership, transport }) => { + const t = this.matrixRTCSession.resolveActiveFocus(membership); + return t && + isLivekitTransport(t) && + t.livekit_service_url !== localServiceUrl + ? [t.livekit_service_url] + : []; + }), + ); + + for (const remoteServiceUrl of remoteServiceUrls) { + let nextConnection = prev.get(remoteServiceUrl); + if (!nextConnection) { + logger.log( + "SFU remoteConnections$ construct new connection: ", + remoteServiceUrl, + ); + nextConnection = new Connection( + { + livekit_service_url: remoteServiceUrl, + livekit_alias: this.livekitAlias, + type: "livekit", + }, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + this.remoteTransports$, + this.e2eeLivekitOptions(), + ); + } else { + logger.log( + "SFU remoteConnections$ use prev connection: ", + remoteServiceUrl, + ); } + next.set(remoteServiceUrl, nextConnection); } + } - for (const connection of stopped.values()) connection.stop(); - return next; - }, - ), + return next; + }), + map((transports) => [...transports.values()]), ), ); - private readonly join$ = new Subject(); + /** + * A list of the connections that should be active at any given time. + */ + private readonly connections$ = this.scope.behavior( + combineLatest( + [this.localConnection$, this.remoteConnections$], + (local, remote) => [ + ...(local?.state === "ready" ? [local.value] : []), + ...remote.values(), + ], + ), + ); - public join(): void { - this.join$.next(); - } - - private readonly connectionInstructions$ = this.join$.pipe( - switchMap(() => this.remoteConnections$), - startWith(new Map()), + private readonly connectionInstructions$ = this.connections$.pipe( pairwise(), map(([prev, next]) => { const start = new Set(next.values()); - for (const connection of prev.values()) start.delete(connection); + for (const connection of prev) start.delete(connection); const stop = new Set(prev.values()); - for (const connection of next.values()) stop.delete(connection); + for (const connection of next) stop.delete(connection); return { start, stop }; }), this.scope.share, ); + /** + * Emits with a connection whenever it should be started. + */ private readonly startConnection$ = this.connectionInstructions$.pipe( concatMap(({ start }) => start), ); + /** + * Emits with a connection whenever it should be stopped. + */ private readonly stopConnection$ = this.connectionInstructions$.pipe( concatMap(({ stop }) => stop), ); public readonly allLivekitRooms$ = this.scope.behavior( - combineLatest([ - this.remoteConnections$, - this.localConnection, - this.localFocus, - ]).pipe( - map(([remoteConnections, localConnection, localFocus]) => - Array.from(remoteConnections.entries()) - .map( - ([index, c]) => - ({ - room: c.livekitRoom, - url: index, - }) as { room: LivekitRoom; url: string; isLocal?: boolean }, - ) - .concat([ - { - room: localConnection.livekitRoom, - url: localFocus.livekit_service_url, - isLocal: true, - }, - ]), + this.connections$.pipe( + map((connections) => + [...connections.values()].map((c) => ({ + room: c.livekitRoom, + url: c.transport.livekit_service_url, + isLocal: c instanceof PublishConnection, + })), ), - startWith([]), ), ); private readonly userId = this.matrixRoom.client.getUserId(); + private readonly deviceId = this.matrixRoom.client.getDeviceId(); private readonly matrixConnected$ = this.scope.behavior( // To consider ourselves connected to MatrixRTC, we check the following: @@ -679,6 +790,10 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; + /** + * Lists, for each LiveKit room, the LiveKit participants whose media should + * be presented. + */ public readonly participantsByRoom$ = this.scope.behavior< { livekitRoom: LivekitRoom; @@ -689,9 +804,12 @@ export class CallViewModel extends ViewModel { }[]; }[] >( - combineLatest([this.localConnection, this.localFocus]) + // TODO: Move this logic into Connection/PublishConnection if possible + this.localConnectionAndTransport$ .pipe( - switchMap(([localConnection, localFocus]) => { + switchMap((values) => { + if (values?.state !== "ready") return []; + const localConnection = values.value.connection; const memberError = (): never => { throw new Error("No room member for call membership"); }; @@ -702,12 +820,9 @@ export class CallViewModel extends ViewModel { }; return this.remoteConnections$.pipe( - switchMap((connections) => + switchMap((remoteConnections) => combineLatest( - [ - [localFocus.livekit_service_url, localConnection] as const, - ...connections, - ].map(([url, c]) => + [localConnection, ...remoteConnections].map((c) => c.publishingParticipants$.pipe( map((ps) => { const participants: { @@ -726,7 +841,7 @@ export class CallViewModel extends ViewModel { return { livekitRoom: c.livekitRoom, - url, + url: c.transport.livekit_service_url, participants, }; }), @@ -809,12 +924,8 @@ export class CallViewModel extends ViewModel { * List of MediaItems that we want to display */ private readonly mediaItems$ = this.scope.behavior( - combineLatest([ - this.participantsByRoom$, - duplicateTiles.value$, - this.memberships$, - ]).pipe( - scan((prevItems, [participantsByRoom, duplicateTiles, memberships]) => { + combineLatest([this.participantsByRoom$, duplicateTiles.value$]).pipe( + scan((prevItems, [participantsByRoom, duplicateTiles]) => { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { for (const { livekitRoom, participants } of participantsByRoom) { @@ -829,6 +940,7 @@ export class CallViewModel extends ViewModel { if (prevMedia && prevMedia instanceof UserMedia) { prevMedia.updateParticipant(participant); if (prevMedia.vm.member === undefined) { + // TODO-MULTI-SFU: This is outdated. // We have a previous media created because of the `debugShowNonMember` flag. // In this case we actually replace the media item. // This "hack" never occurs if we do not use the `debugShowNonMember` debugging @@ -931,6 +1043,16 @@ export class CallViewModel extends ViewModel { this.memberships$.pipe(map((ms) => ms.length)), ); + private readonly allOthersLeft$ = this.memberships$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.every((m) => m.sender === this.userId) && + prev.some((m) => m.sender !== this.userId), + ), + map(() => {}), + ); + private readonly didSendCallNotification$ = fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification, @@ -1055,56 +1177,12 @@ export class CallViewModel extends ViewModel { map(() => {}), throttleTime(THROTTLE_SOUND_EFFECT_MS), ); - /** - * This observable tracks the matrix users that are currently in the call. - * There can be just one matrix user with multiple participants (see also participantChanges$) - */ - public readonly matrixUserChanges$ = this.userMedia$.pipe( - map( - (mediaItems) => - new Set( - mediaItems - .map((m) => m.vm.member?.userId) - .filter((id) => id !== undefined), - ), - ), - scan< - Set, - { - userIds: Set; - joinedUserIds: Set; - leftUserIds: Set; - } - >( - (prevState, userIds) => { - const left = new Set( - [...prevState.userIds].filter((id) => !userIds.has(id)), - ); - const joined = new Set( - [...userIds].filter((id) => !prevState.userIds.has(id)), - ); - return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; - }, - { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, - ), - ); - - private readonly allOthersLeft$ = this.matrixUserChanges$.pipe( - filter(({ userIds, leftUserIds }) => { - if (!this.userId) { - logger.warn("Could not access user ID to compute allOthersLeft"); - return false; - } - return ( - userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 - ); - }), - map(() => "allOthersLeft" as const), - ); // Public for testing public readonly autoLeave$ = merge( - this.options.autoLeaveWhenOthersLeft ? this.allOthersLeft$ : NEVER, + this.options.autoLeaveWhenOthersLeft + ? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) + : NEVER, this.callPickupState$.pipe( filter((state) => state === "timeout" || state === "decline"), ), @@ -1132,6 +1210,9 @@ export class CallViewModel extends ViewModel { merge(this.userHangup$, this.widgetHangup$).pipe( map(() => "user" as const), ), + ).pipe( + this.scope.share, + tap((reason) => this.leaveHoisted$.next(reason)), ); /** @@ -1820,9 +1901,12 @@ export class CallViewModel extends ViewModel { * Whether we are sharing our screen. */ public readonly sharingScreen$ = this.scope.behavior( - from(this.localConnection).pipe( - switchMap((c) => sharingScreen$(c.livekitRoom.localParticipant)), - startWith(false), + from(this.localConnection$).pipe( + switchMap((c) => + c?.state === "ready" + ? sharingScreen$(c.value.livekitRoom.localParticipant) + : of(false), + ), ), ); @@ -1834,17 +1918,26 @@ export class CallViewModel extends ViewModel { "getDisplayMedia" in (navigator.mediaDevices ?? {}) && !this.urlParams.hideScreensharing ? (): void => - void this.localConnection.then( - (c) => - void c.livekitRoom.localParticipant - .setScreenShareEnabled(!this.sharingScreen$.value, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error), - ) + // Once a connection is ready... + void this.localConnection$ + .pipe( + takeWhile((c) => c !== null && c.state !== "error"), + switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)), + take(1), + this.scope.bind(), + ) + // ...toggle screen sharing. + .subscribe( + (c) => + void c.livekitRoom.localParticipant + .setScreenShareEnabled(!this.sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error), + ) : null; public constructor( @@ -1864,32 +1957,33 @@ export class CallViewModel extends ViewModel { ) { super(); - void from(this.localConnection) - .pipe(this.scope.bind()) - .subscribe( - (c) => - void c - .start() - // eslint-disable-next-line no-console - .then(() => console.log("successfully started publishing")) - // eslint-disable-next-line no-console - .catch((e) => console.error("failed to start publishing", e)), - ); + // Start and stop local and remote connections as needed + this.startConnection$.pipe(this.scope.bind()).subscribe( + (c) => + void c.start().then( + () => logger.info(`Connected to ${c.transport.livekit_service_url}`), + (e) => + logger.error( + `Failed to start connection to ${c.transport.livekit_service_url}`, + e, + ), + ), + ); + this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => { + logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); + c.stop(); + }); - this.startConnection$ - .pipe(this.scope.bind()) - .subscribe((c) => void c.start()); - this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => c.stop()); - - combineLatest([this.localFocus, this.join$]) - .pipe(this.scope.bind()) - .subscribe(([localFocus]) => { + // Start and stop session membership as needed + this.localTransport$.pipe(this.scope.bind()).subscribe((localTransport) => { + if (localTransport?.state === "ready") { void enterRTCSession( this.matrixRTCSession, - localFocus, + localTransport.value, this.options.encryptionSystem.kind !== E2eeType.NONE, true, true, + multiSfu.value$.value, ) .catch((e) => logger.error("Error entering RTC session", e)) .then(() => @@ -1906,19 +2000,20 @@ export class CallViewModel extends ViewModel { ), ), ); - }); - this.leave$.pipe(this.scope.bind()).subscribe(() => { - // Only sends Matrix leave event. The LiveKit session will disconnect once, uh... - // (TODO-MULTI-SFU does anything actually cause it to disconnect?) - void this.matrixRTCSession - .leaveRoomSession() - .catch((e) => logger.error("Error leaving RTC session", e)) - .then(async () => - widget?.api.transport - .send(ElementWidgetActions.HangupCall, {}) - .catch((e) => logger.error("Failed to send hangup action", e)), - ); + return (): void => + // Only sends Matrix leave event. The LiveKit session will disconnect + // as soon as either the stopConnection$ handler above gets to it or + // the view model is destroyed. + void this.matrixRTCSession + .leaveRoomSession() + .catch((e) => logger.error("Error leaving RTC session", e)) + .then(async () => + widget?.api.transport + .send(ElementWidgetActions.HangupCall, {}) + .catch((e) => logger.error("Failed to send hangup action", e)), + ); + } }); // Pause upstream of all local media tracks when we're disconnected from @@ -1927,10 +2022,12 @@ export class CallViewModel extends ViewModel { // We use matrixConnected$ rather than reconnecting$ because we want to // pause tracks during the initial joining sequence too until we're sure // that our own media is displayed on screen. - void this.localConnection.then((localConnection) => - this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { + combineLatest([this.localConnection$, this.matrixConnected$]) + .pipe(this.scope.bind()) + .subscribe(([connection, connected]) => { + if (connection?.state !== "ready") return; const publications = - localConnection.livekitRoom.localParticipant.trackPublications.values(); + connection.value.livekitRoom.localParticipant.trackPublications.values(); if (connected) { for (const p of publications) { if (p.track?.isUpstreamPaused === true) { @@ -1966,8 +2063,7 @@ export class CallViewModel extends ViewModel { } } } - }), - ); + }); // Join automatically this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 8eaed463..992d8840 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -62,7 +62,7 @@ export class Connection { protected readonly sfuConfig = getSFUConfigWithOpenID( this.client, - this.focus.livekit_service_url, + this.transport.livekit_service_url, this.livekitAlias, ); @@ -72,12 +72,12 @@ export class Connection { public connectionState$: Behavior; public constructor( - protected readonly focus: LivekitTransport, + public readonly transport: LivekitTransport, protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, - protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitTransport }[] + protected readonly remoteTransports$: Behavior< + { membership: CallMembership; transport: LivekitTransport }[] >, e2eeLivekitOptions: E2EEOptions | undefined, livekitRoom: LivekitRoom | undefined = undefined, @@ -95,12 +95,13 @@ export class Connection { this.publishingParticipants$ = this.scope.behavior( combineLatest( - [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], - (participants, membershipsFocusMap) => - membershipsFocusMap + [this.participantsIncludingSubscribers$, this.remoteTransports$], + (participants, remoteTransports) => + remoteTransports // Find all members that claim to publish on this connection - .flatMap(({ membership, focus }) => - focus.livekit_service_url === this.focus.livekit_service_url + .flatMap(({ membership, transport }) => + transport.livekit_service_url === + this.transport.livekit_service_url ? [membership] : [], ) @@ -130,23 +131,35 @@ export class PublishConnection extends Connection { if (!this.stopped) await this.livekitRoom.connect(url, jwt); if (!this.stopped) { - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value, - }); - for (const track of tracks) { - await this.livekitRoom.localParticipant.publishTrack(track); + // TODO-MULTI-SFU: Prepublish a microphone track + const audio = this.muteStates.audio.enabled$.value; + const video = this.muteStates.video.enabled$.value; + // createTracks throws if called with audio=false and video=false + if (audio || video) { + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio, + video, + }); + for (const track of tracks) { + await this.livekitRoom.localParticipant.publishTrack(track); + } } } } + public stop(): void { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + super.stop(); + } + public constructor( - focus: LivekitTransport, + transport: LivekitTransport, livekitAlias: string, client: MatrixClient, scope: ObservableScope, - membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitTransport }[] + remoteTransports$: Behavior< + { membership: CallMembership; transport: LivekitTransport }[] >, devices: MediaDevices, private readonly muteStates: MuteStates, @@ -182,11 +195,11 @@ export class PublishConnection extends Connection { }); super( - focus, + transport, livekitAlias, client, scope, - membershipsFocusMap$, + remoteTransports$, e2eeLivekitOptions, room, ); @@ -218,10 +231,6 @@ export class PublishConnection extends Connection { } return this.livekitRoom.localParticipant.isCameraEnabled; }); - this.scope.onEnd(() => { - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - }); const syncDevice = ( kind: MediaDeviceKind, diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index c93e88d8..07bc5665 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -137,7 +137,7 @@ export class MuteStates { this.scope, this.mediaDevices.audioInput, this.joined$, - Config.get().media_devices.enable_video, + Config.get().media_devices.enable_audio, ); public readonly video = new MuteState( this.scope, diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index fe99d89b..8ac816ca 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject, distinctUntilChanged, + filter, type Observable, share, - Subject, + take, takeUntil, } from "rxjs"; @@ -24,9 +25,11 @@ const nothing = Symbol("nothing"); * A scope which limits the execution lifetime of its bound Observables. */ export class ObservableScope { - private readonly ended$ = new Subject(); + private readonly ended$ = new BehaviorSubject(false); - private readonly bindImpl: MonoTypeOperator = takeUntil(this.ended$); + private readonly bindImpl: MonoTypeOperator = takeUntil( + this.ended$.pipe(filter((ended) => ended)), + ); /** * Binds an Observable to this scope, so that it completes when the scope @@ -78,15 +81,19 @@ export class ObservableScope { * Ends the scope, causing any bound Observables to complete. */ public end(): void { - this.ended$.next(); - this.ended$.complete(); + this.ended$.next(true); } /** * Register a callback to be executed when the scope is ended. */ public onEnd(callback: () => void): void { - this.ended$.subscribe(callback); + this.ended$ + .pipe( + filter((ended) => ended), + take(1), + ) + .subscribe(callback); } } diff --git a/src/utils/errors.ts b/src/utils/errors.ts index 5cb0b450..b77c0ff0 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -11,7 +11,7 @@ export enum ErrorCode { /** * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured. */ - MISSING_MATRIX_RTC_FOCUS = "MISSING_MATRIX_RTC_FOCUS", + MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT", CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR", /** LiveKit indicates that the server has hit its track limits */ INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", @@ -54,18 +54,18 @@ export class ElementCallError extends Error { } } -export class MatrixRTCFocusMissingError extends ElementCallError { +export class MatrixRTCTransportMissingError extends ElementCallError { public domain: string; public constructor(domain: string) { super( t("error.call_is_not_supported"), - ErrorCode.MISSING_MATRIX_RTC_FOCUS, + ErrorCode.MISSING_MATRIX_RTC_TRANSPORT, ErrorCategory.CONFIGURATION_ISSUE, - t("error.matrix_rtc_focus_missing", { + t("error.matrix_rtc_transport_missing", { domain, brand: import.meta.env.VITE_PRODUCT_NAME || "Element Call", - errorCode: ErrorCode.MISSING_MATRIX_RTC_FOCUS, + errorCode: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT, }), ); this.domain = domain; From 1820cac3f66e9b9801209990d5ad5469ad21e97a Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Oct 2025 19:14:48 -0400 Subject: [PATCH 056/144] Create media items for session members not joined to LiveKit --- src/state/CallViewModel.ts | 49 ++++++++++++++----------------------- src/state/Connection.ts | 14 +++++------ src/state/MediaViewModel.ts | 10 ++++---- src/tile/MediaView.tsx | 2 +- src/tile/SpotlightTile.tsx | 2 +- 5 files changed, 32 insertions(+), 45 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8988e518..6e333bec 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -304,7 +304,7 @@ class UserMedia { public readonly presenter$: Behavior; public constructor( public readonly id: string, - member: RoomMember | undefined, + member: RoomMember, participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -377,7 +377,7 @@ class ScreenShare { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -799,7 +799,8 @@ export class CallViewModel extends ViewModel { livekitRoom: LivekitRoom; url: string; participants: { - participant: LocalParticipant | RemoteParticipant; + id: string; + participant: LocalParticipant | RemoteParticipant | undefined; member: RoomMember; }[]; }[] @@ -814,6 +815,7 @@ export class CallViewModel extends ViewModel { throw new Error("No room member for call membership"); }; const localParticipant = { + id: "local", participant: localConnection.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), @@ -826,9 +828,14 @@ export class CallViewModel extends ViewModel { c.publishingParticipants$.pipe( map((ps) => { const participants: { - participant: LocalParticipant | RemoteParticipant; + id: string; + participant: + | LocalParticipant + | RemoteParticipant + | undefined; member: RoomMember; }[] = ps.map(({ participant, membership }) => ({ + id: `${membership.sender}:${membership.deviceId}`, participant, member: getRoomMemberFromRtcMember( @@ -929,26 +936,12 @@ export class CallViewModel extends ViewModel { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { for (const { livekitRoom, participants } of participantsByRoom) { - for (const { participant, member } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; - + for (const { id, participant, member } of participants) { for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { + const mediaId = `${id}:${i}`; + const prevMedia = prevItems.get(mediaId); + if (prevMedia instanceof UserMedia) prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // TODO-MULTI-SFU: This is outdated. - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } - } yield [ mediaId, @@ -965,14 +958,10 @@ export class CallViewModel extends ViewModel { this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), + map((m) => m.get(id) ?? "[👻]"), ), + this.handsRaised$.pipe(map((v) => v[id]?.time ?? null)), + this.reactions$.pipe(map((v) => v[id] ?? undefined)), ), ]; @@ -989,7 +978,7 @@ export class CallViewModel extends ViewModel { livekitRoom, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), + map((m) => m.get(id) ?? "[👻]"), ), ), ]; diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 992d8840..4908e42f 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -66,7 +66,7 @@ export class Connection { this.livekitAlias, ); - public readonly participantsIncludingSubscribers$; + private readonly participantsIncludingSubscribers$; public readonly publishingParticipants$; public readonly livekitRoom: LivekitRoom; @@ -105,13 +105,11 @@ export class Connection { ? [membership] : [], ) - // Find all associated publishing livekit participant objects - .flatMap((membership) => { - const participant = participants.find( - (p) => - p.identity === `${membership.sender}:${membership.deviceId}`, - ); - return participant ? [{ participant, membership }] : []; + // Pair with their associated LiveKit participant (if any) + .map((membership) => { + const id = `${membership.sender}:${membership.deviceId}`; + const participant = participants.find((p) => p.identity === id); + return { participant, membership }; }), ), [], diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index dc2c135a..016c6a49 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -255,7 +255,7 @@ abstract class BaseMediaViewModel extends ViewModel { */ // TODO: Fully separate the data layer from the UI layer by keeping the // member object internal - public readonly member: RoomMember | undefined, + public readonly member: RoomMember, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -403,7 +403,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -535,7 +535,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -641,7 +641,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -736,7 +736,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index a4fd0402..8506a650 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -32,7 +32,7 @@ interface Props extends ComponentProps { video: TrackReferenceOrPlaceholder | undefined; videoFit: "cover" | "contain"; mirror: boolean; - member: RoomMember | undefined; + member: RoomMember; videoEnabled: boolean; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 663fb912..b1a15332 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -55,7 +55,7 @@ interface SpotlightItemBaseProps { targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; videoEnabled: boolean; - member: RoomMember | undefined; + member: RoomMember; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; displayName: string; From 1fff71ace1f11dcc869fdfc301114f9f77f67d9a Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Oct 2025 21:00:45 -0400 Subject: [PATCH 057/144] Actually leave the MatrixRTC session again --- src/state/CallViewModel.ts | 123 +++++++++++++++++------------------ src/state/Connection.ts | 35 +++++----- src/state/ObservableScope.ts | 38 +++++++++++ 3 files changed, 116 insertions(+), 80 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6e333bec..1a20589c 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -37,7 +37,6 @@ import { Subject, combineLatest, concat, - concatMap, distinctUntilChanged, endWith, filter, @@ -678,6 +677,9 @@ export class CallViewModel extends ViewModel { ), ); + /** + * Emits with connections whenever they should be started or stopped. + */ private readonly connectionInstructions$ = this.connections$.pipe( pairwise(), map(([prev, next]) => { @@ -688,20 +690,6 @@ export class CallViewModel extends ViewModel { return { start, stop }; }), - this.scope.share, - ); - - /** - * Emits with a connection whenever it should be started. - */ - private readonly startConnection$ = this.connectionInstructions$.pipe( - concatMap(({ start }) => start), - ); - /** - * Emits with a connection whenever it should be stopped. - */ - private readonly stopConnection$ = this.connectionInstructions$.pipe( - concatMap(({ stop }) => stop), ); public readonly allLivekitRooms$ = this.scope.behavior( @@ -1947,61 +1935,70 @@ export class CallViewModel extends ViewModel { super(); // Start and stop local and remote connections as needed - this.startConnection$.pipe(this.scope.bind()).subscribe( - (c) => - void c.start().then( - () => logger.info(`Connected to ${c.transport.livekit_service_url}`), - (e) => - logger.error( - `Failed to start connection to ${c.transport.livekit_service_url}`, - e, - ), - ), - ); - this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => { - logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); - c.stop(); - }); - - // Start and stop session membership as needed - this.localTransport$.pipe(this.scope.bind()).subscribe((localTransport) => { - if (localTransport?.state === "ready") { - void enterRTCSession( - this.matrixRTCSession, - localTransport.value, - this.options.encryptionSystem.kind !== E2eeType.NONE, - true, - true, - multiSfu.value$.value, - ) - .catch((e) => logger.error("Error entering RTC session", e)) - .then(() => - // Update our member event when our mute state changes. - this.muteStates.video.enabled$ - .pipe(this.scope.bind(), takeUntil(this.leave$)) - // eslint-disable-next-line rxjs/no-nested-subscribe - .subscribe( - (videoEnabled) => - // TODO: Ensure that these calls are serialized in case of - // fast video toggling - void this.matrixRTCSession.updateCallIntent( - videoEnabled ? "video" : "audio", - ), + this.connectionInstructions$ + .pipe(this.scope.bind()) + .subscribe(({ start, stop }) => { + for (const c of stop) { + logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); + c.stop(); + } + for (const c of start) { + c.start().then( + () => + logger.info(`Connected to ${c.transport.livekit_service_url}`), + (e) => + logger.error( + `Failed to start connection to ${c.transport.livekit_service_url}`, + e, ), ); + } + }); - return (): void => + // Start and stop session membership as needed + this.scope.reconcile(this.localTransport$, async (localTransport) => { + if (localTransport?.state === "ready") { + try { + await enterRTCSession( + this.matrixRTCSession, + localTransport.value, + this.options.encryptionSystem.kind !== E2eeType.NONE, + true, + true, + multiSfu.value$.value, + ); + } catch (e) { + logger.error("Error entering RTC session", e); + } + // Update our member event when our mute state changes. + const muteSubscription = this.muteStates.video.enabled$.subscribe( + (videoEnabled) => + // TODO: Ensure that these calls are serialized in case of + // fast video toggling + void this.matrixRTCSession.updateCallIntent( + videoEnabled ? "video" : "audio", + ), + ); + + return async (): Promise => { + muteSubscription.unsubscribe(); // Only sends Matrix leave event. The LiveKit session will disconnect // as soon as either the stopConnection$ handler above gets to it or // the view model is destroyed. - void this.matrixRTCSession - .leaveRoomSession() - .catch((e) => logger.error("Error leaving RTC session", e)) - .then(async () => - widget?.api.transport - .send(ElementWidgetActions.HangupCall, {}) - .catch((e) => logger.error("Failed to send hangup action", e)), + try { + await this.matrixRTCSession.leaveRoomSession(); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + try { + await widget?.api.transport.send( + ElementWidgetActions.HangupCall, + {}, ); + } catch (e) { + logger.error("Failed to send hangup action", e); + } + }; } }); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 4908e42f..55afdacf 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -125,6 +125,24 @@ export class Connection { export class PublishConnection extends Connection { public async start(): Promise { this.stopped = false; + + this.muteStates.audio.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit audio input mute state", e); + } + return this.livekitRoom.localParticipant.isMicrophoneEnabled; + }); + this.muteStates.video.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setCameraEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit video input mute state", e); + } + return this.livekitRoom.localParticipant.isCameraEnabled; + }); + const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); @@ -213,23 +231,6 @@ export class PublishConnection extends Connection { ); trackProcessorSync(track$, trackerProcessorState$); - this.muteStates.audio.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); - } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; - }); - this.muteStates.video.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); - } - return this.livekitRoom.localParticipant.isCameraEnabled; - }); - const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 8ac816ca..08a4b859 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -7,7 +7,10 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject, + catchError, distinctUntilChanged, + EMPTY, + endWith, filter, type Observable, share, @@ -95,6 +98,41 @@ export class ObservableScope { ) .subscribe(callback); } + + // TODO-MULTI-SFU Dear Future Robin, please document this. Love, Past Robin. + public reconcile( + value$: Behavior, + callback: (value: T) => Promise<(() => Promise) | undefined>, + ): void { + let latestValue: T | typeof nothing = nothing; + let reconciledValue: T | typeof nothing = nothing; + let cleanUp: (() => Promise) | undefined = undefined; + let callbackPromise: Promise<(() => Promise) | undefined>; + value$ + .pipe( + catchError(() => EMPTY), + this.bind(), + endWith(nothing), + ) + .subscribe((value) => { + void (async (): Promise => { + if (latestValue === nothing) { + latestValue = value; + while (latestValue !== reconciledValue) { + await cleanUp?.(); + reconciledValue = latestValue; + if (latestValue !== nothing) { + callbackPromise = callback(latestValue); + cleanUp = await callbackPromise; + } + } + latestValue = nothing; + } else { + latestValue = value; + } + })(); + }); + } } /** From 91a366fa2a92de9853bbae464c17c44a229686d3 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 6 Oct 2025 10:50:10 +0200 Subject: [PATCH 058/144] tests: Publish connection states --- src/state/Connection.test.ts | 188 +++++++++++++++++++++++++++++++---- 1 file changed, 166 insertions(+), 22 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 5529e588..5f1778b0 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -5,18 +5,30 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, describe, expect, it, type MockedObject, vi } from "vitest"; +import { afterEach, describe, expect, it, type Mock, Mocked, type MockedObject, vi } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; -import { ConnectionState, type RemoteParticipant, type Room as LivekitRoom, RoomEvent } from "livekit-client"; +import { BehaviorSubject, of } from "rxjs"; +import { + ConnectionState, + type LocalParticipant, + type RemoteParticipant, + type Room as LivekitRoom, + RoomEvent, type RoomOptions +} from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; +import { type BackgroundOptions, type ProcessorWrapper } from "@livekit/track-processors"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../utils/errors.ts"; +import { PublishConnection } from "./PublishConnection.ts"; +import { mockMediaDevices, mockMuteStates } from "../utils/test.ts"; +import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; +import { type MuteStates } from "./MuteStates.ts"; +import { DeviceLabel, MediaDevice, SelectedDevice } from "./MediaDevices.ts"; let testScope: ObservableScope; @@ -25,6 +37,9 @@ let client: MockedObject; let fakeLivekitRoom: MockedObject; +let localParticipantEventEmiter: EventEmitter; +let fakeLocalParticipant: MockedObject; + let fakeRoomEventEmiter: EventEmitter; let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; @@ -49,18 +64,32 @@ function setupTest(): void { } as unknown as OpenIDClientParts); fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + localParticipantEventEmiter = new EventEmitter(); + + fakeLocalParticipant = vi.mocked({ + identity: "@me:example.org", + isMicrophoneEnabled: vi.fn().mockReturnValue(true), + getTrackPublication: vi.fn().mockReturnValue(undefined), + on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter), + off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter), + addListener: localParticipantEventEmiter.addListener.bind(localParticipantEventEmiter), + removeListener: localParticipantEventEmiter.removeListener.bind(localParticipantEventEmiter), + removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(localParticipantEventEmiter) + } as unknown as LocalParticipant); fakeRoomEventEmiter = new EventEmitter(); fakeLivekitRoom = vi.mocked({ connect: vi.fn(), disconnect: vi.fn(), remoteParticipants: new Map(), + localParticipant: fakeLocalParticipant, state: ConnectionState.Disconnected, on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter) + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + setE2EEEnabled: vi.fn().mockResolvedValue(undefined) } as unknown as LivekitRoom); } @@ -424,7 +453,7 @@ function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { return vi.mocked({ sender: userId, - deviceId: deviceId, + deviceId: deviceId } as unknown as CallMembership); } @@ -440,19 +469,19 @@ describe("Publishing participants observations", () => { const danIsAPublisher = Promise.withResolvers(); const observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; connection.publishingParticipants$.subscribe((publishers) => { - observedPublishers.push(publishers); - if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { - bobIsAPublisher.resolve(); - } - if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { - danIsAPublisher.resolve(); - } + observedPublishers.push(publishers); + if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } }); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. - let participants: RemoteParticipant[]= [ + let participants: RemoteParticipant[] = [ fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), @@ -477,7 +506,7 @@ describe("Publishing participants observations", () => { livekit_alias: "!roomID:example.org", livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", type: "livekit" - } + }; const rtcMemberships = [ @@ -485,7 +514,7 @@ describe("Publishing participants observations", () => { { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, // Alice and carol is on a different focus { membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), focus: otherFocus }, - { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), focus: otherFocus }, + { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), focus: otherFocus } // NO DAVE YET ]; // signal this change in rtc memberships @@ -520,7 +549,7 @@ describe("Publishing participants observations", () => { const updatedPublishers = observedPublishers.pop(); expect(updatedPublishers?.length).toEqual(1); expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); - }) + }); it("should be scoped to parent scope", async () => { @@ -533,8 +562,8 @@ describe("Publishing participants observations", () => { observedPublishers.push(publishers); }); - let participants: RemoteParticipant[]= [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + let participants: RemoteParticipant[] = [ + fakeRemoteLivekitParticipant("@bob:example.org:DEV111") ]; // Let's simulate 3 members on the livekitRoom @@ -552,7 +581,7 @@ describe("Publishing participants observations", () => { const rtcMemberships = [ // Say bob is on the same focus - { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, + { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus } ]; // signal this change in rtc memberships fakeMembershipsFocusMap$.next(rtcMemberships); @@ -575,7 +604,122 @@ describe("Publishing participants observations", () => { fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); expect(observedPublishers.length).toEqual(0); - }) - - + }); +}); + + +describe("PublishConnection", () => { + + let fakeBlurProcessor: ProcessorWrapper; + let roomFactoryMock: Mock<() => LivekitRoom>; + let muteStates: MockedObject; + + function setUpPublishConnection() { + setupTest(); + + roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); + + + muteStates = mockMuteStates(); + + fakeBlurProcessor = vi.mocked>({ + name: "BackgroundBlur", + start: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + setOptions: vi.fn().mockResolvedValue(undefined), + getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), + isRunning: vi.fn().mockReturnValue(false) + }); + + + } + + + describe("Livekit room creation", () => { + + + function createSetup() { + setUpPublishConnection(); + + const fakeTrackProcessorSubject$ = new BehaviorSubject({ + supported: true, + processor: undefined + }); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: roomFactoryMock + }; + + const audioInput = { + available$: of(new Map([["mic1", { id: "mic1" }]])), + selected$: new BehaviorSubject({ id: "mic1" }), + select(): void { + } + }; + + const videoInput = { + available$: of(new Map([["cam1", { id: "cam1" }]])), + selected$: new BehaviorSubject({ id: "cam1" }), + select(): void { + } + }; + + const audioOutput = { + available$: of(new Map([["speaker", { id: "speaker" }]])), + selected$: new BehaviorSubject({ id: "speaker" }), + select(): void { + } + }; + + const fakeDevices = mockMediaDevices({ + audioInput, + videoInput, + audioOutput + }); + + new PublishConnection( + opts, + fakeDevices, + muteStates, + undefined, + fakeTrackProcessorSubject$ + ); + + } + + it("should create room with proper initial audio and video settings", () => { + + createSetup(); + + expect(roomFactoryMock).toHaveBeenCalled(); + + const lastCallArgs = roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; + + const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; + expect(roomOptions).toBeDefined(); + + expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); + expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); + expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); + + }); + + it("respect controlledAudioDevices", () => { + // TODO: Refactor the code to make it testable. + // The UrlParams module is a singleton has a cache and is very hard to test. + // This breaks other tests as well if not handled properly. + // vi.mock(import("./../UrlParams"), () => { + // return { + // getUrlParams: vi.fn().mockReturnValue({ + // controlledAudioDevices: true + // }) + // }; + // }); + + }); + }); }); From c3c0516f0d663b18ec3e84e3ee7c604743ba58de Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:00:59 +0200 Subject: [PATCH 059/144] Lint: fix all the lint errors --- src/livekit/MatrixAudioRenderer.test.tsx | 89 ++++++++++++++++--- src/livekit/MatrixAudioRenderer.tsx | 6 +- src/main.tsx | 1 - src/room/CallEventAudioRenderer.test.tsx | 3 +- src/room/GroupCallView.test.tsx | 7 +- src/room/InCallView.test.tsx | 9 +- src/room/InCallView.tsx | 6 +- src/room/VideoPreview.test.tsx | 7 +- src/rtcSessionHelpers.test.ts | 104 ++++++++++++++--------- src/state/Async.ts | 14 +-- src/state/CallViewModel.test.ts | 7 +- src/state/CallViewModel.ts | 9 +- src/state/Connection.test.ts | 12 +-- src/state/MuteStates.ts | 11 ++- src/tile/MediaView.test.tsx | 7 +- src/utils/test.ts | 11 ++- 16 files changed, 206 insertions(+), 97 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 4fe7d333..e2464eb8 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { render } from "@testing-library/react"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { getTrackReferenceId, type TrackReference, @@ -15,11 +14,19 @@ import { import { type RemoteAudioTrack } from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; +import { of } from "rxjs"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { mockMediaDevices, mockTrack } from "../utils/test"; +import { + mockLivekitRoom, + mockMatrixRoomMember, + mockMediaDevices, + mockRtcMembership, + mockTrack +} from "../utils/test"; + export const TestAudioContextConstructor = vi.fn(() => testAudioContext); @@ -52,10 +59,26 @@ const tracks = [mockTrack("test:123")]; vi.mocked(useTracks).mockReturnValue(tracks); it("should render for member", () => { + // TODO this is duplicated test setup in all tests + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); const { container, queryAllByTestId } = render( , ); @@ -64,12 +87,29 @@ it("should render for member", () => { }); it("should not render without member", () => { - const memberships = [ - { sender: "othermember", deviceId: "123" }, - ] as CallMembership[]; + // const memberships = [ + // { sender: "othermember", deviceId: "123" }, + // ] as CallMembership[]; + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); const { container, queryAllByTestId } = render( - + , ); expect(container).toBeTruthy(); @@ -77,10 +117,25 @@ it("should not render without member", () => { }); it("should not setup audioContext gain and pan if there is no need to.", () => { + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); render( , ); @@ -100,11 +155,25 @@ it("should setup audioContext gain and pan", () => { pan: 1, volume: 0.1, }); + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); render( + participants={[p]} + url={""} + livekitRoom={livekitRoom} /> , ); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index f402b32d..76c206c7 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -33,7 +33,9 @@ export interface MatrixAudioRendererProps { * that are not expected to be in the rtc session. */ participants: { - participant: Participant; + id: string; + // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit? + participant: Participant | undefined; member: RoomMember; }[]; /** @@ -82,7 +84,7 @@ export function LivekitRoomAudioRenderer({ if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, - `current members: ${participants.map((p) => p.participant.identity)}`, + `current members: ${participants.map((p) => p.participant?.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); diff --git a/src/main.tsx b/src/main.tsx index e795a13c..f27b55a4 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -11,7 +11,6 @@ Please see LICENSE in the repository root for full details. // dependency references. import "matrix-js-sdk/lib/browser-index"; -import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import "./index.css"; import { logger } from "matrix-js-sdk/lib/logger"; diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 40b79da4..e7d7e85a 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -155,7 +155,8 @@ test("plays one sound when a hand is raised", () => { act(() => { handRaisedSubject$.next({ - [bobRtcMember.callId]: { + // TODO: What is this string supposed to be? + [`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: { time: new Date(), membershipEventId: "", reactionEventId: "", diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index b8bc2f53..22d99b31 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -26,7 +26,6 @@ import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-cont import { useState } from "react"; import { TooltipProvider } from "@vector-im/compound-web"; -import { type MuteStates } from "./MuteStates"; import { prefetchSounds } from "../soundUtils"; import { useAudioContext } from "../useAudioContext"; import { ActiveCall } from "./InCallView"; @@ -47,6 +46,7 @@ import { ProcessorProvider } from "../livekit/TrackProcessorContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; import { constant } from "../state/Behavior"; +import { type MuteStates } from "../state/MuteStates.ts"; vi.mock("../soundUtils"); vi.mock("../useAudioContext"); @@ -150,7 +150,7 @@ function createGroupCallView( const muteState = { audio: { enabled: false }, video: { enabled: false }, - } as MuteStates; + } as unknown as MuteStates; const { getByText } = render( @@ -164,9 +164,10 @@ function createGroupCallView( skipLobby={false} header={HeaderStyle.Standard} rtcSession={rtcSession as unknown as MatrixRTCSession} - isJoined={joined} muteStates={muteState} widget={widget} + joined={true} + setJoined={function(value: boolean): void { }} /> diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 6d2aaf0a..d2694120 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -24,7 +24,6 @@ import { TooltipProvider } from "@vector-im/compound-web"; import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; -import { type MuteStates } from "./MuteStates"; import { InCallView } from "./InCallView"; import { mockLivekitRoom, @@ -48,6 +47,7 @@ import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement"; import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; +import { type MuteStates } from "../state/MuteStates.ts"; // vi.hoisted(() => { // localStorage = {} as unknown as Storage; @@ -136,7 +136,7 @@ function createInCallView(): RenderResult & { const muteState = { audio: { enabled: false }, video: { enabled: false }, - } as MuteStates; + } as unknown as MuteStates; const livekitRoom = mockLivekitRoom( { localParticipant, @@ -176,11 +176,6 @@ function createInCallView(): RenderResult & { }, }} matrixRoom={room} - livekitRoom={livekitRoom} - participantCount={0} - onLeft={function (): void { - throw new Error("Function not implemented."); - }} onShareClick={null} /> diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 57873b40..8474c2fd 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -23,7 +23,7 @@ import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; -import { useObservable, useObservableEagerState } from "observable-hooks"; +import { useObservable } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { @@ -112,7 +112,6 @@ import { prefetchSounds } from "../soundUtils"; import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; -import { ConnectionLostError } from "../utils/errors.ts"; import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; const maxTapDurationMs = 400; @@ -206,7 +205,8 @@ export const InCallView: FC = ({ useReactionsSender(); useWakeLock(); - const connectionState = useObservableEagerState(vm.livekitConnectionState$); + // TODO multi-sfu This is unused now?? + // const connectionState = useObservableEagerState(vm.livekitConnectionState$); // annoyingly we don't get the disconnection reason this way, // only by listening for the emitted event diff --git a/src/room/VideoPreview.test.tsx b/src/room/VideoPreview.test.tsx index 717333ee..17a05e34 100644 --- a/src/room/VideoPreview.test.tsx +++ b/src/room/VideoPreview.test.tsx @@ -5,12 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, describe, it, vi, beforeAll } from "vitest"; +import { expect, describe, it, beforeAll } from "vitest"; import { render } from "@testing-library/react"; import { type MatrixInfo, VideoPreview } from "./VideoPreview"; import { E2eeType } from "../e2ee/e2eeType"; -import { mockMuteStates } from "../utils/test"; describe("VideoPreview", () => { const matrixInfo: MatrixInfo = { @@ -42,7 +41,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, @@ -54,7 +53,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 2ef9e3f1..1058628f 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -23,37 +23,38 @@ vi.mock("./widget", () => ({ ...actualWidget, widget: { api: { - setAlwaysOnScreen: (): void => {}, - transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() }, + setAlwaysOnScreen: (): void => { + }, + transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() } }, - lazyActions: new EventEmitter(), - }, + lazyActions: new EventEmitter() + } })); test("It joins the correct Session", async () => { const focusFromOlderMembership = { type: "livekit", livekit_service_url: "http://my-oldest-member-service-url.com", - livekit_alias: "my-oldest-member-service-alias", + livekit_alias: "my-oldest-member-service-alias" }; const focusConfigFromWellKnown = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com", + livekit_service_url: "http://my-well-known-service-url.com" }; const focusConfigFromWellKnown2 = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url2.com", + livekit_service_url: "http://my-well-known-service-url2.com" }; const clientWellKnown = { "org.matrix.msc4143.rtc_foci": [ focusConfigFromWellKnown, - focusConfigFromWellKnown2, - ], + focusConfigFromWellKnown2 + ] }; mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, + livekit: { livekit_service_url: "http://my-default-service-url.com" } }); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation( @@ -62,7 +63,7 @@ test("It joins the correct Session", async () => { return Promise.resolve(clientWellKnown); } return Promise.resolve({}); - }, + } ); const mockedSession = vi.mocked({ @@ -74,58 +75,64 @@ test("It joins the correct Session", async () => { access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000, - }), - }, + expires_in: 10000 + }) + } }, memberships: [], getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), getOldestMembership: vi.fn().mockReturnValue({ - getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), + getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]) }), - joinRoomSession: vi.fn(), + joinRoomSession: vi.fn() }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, false); + + await enterRTCSession(mockedSession, { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit" + }, + true); expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( [ { livekit_alias: "my-oldest-member-service-alias", livekit_service_url: "http://my-oldest-member-service-url.com", - type: "livekit", + type: "livekit" }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit", + type: "livekit" }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url2.com", - type: "livekit", + type: "livekit" }, { livekit_alias: "roomId", livekit_service_url: "http://my-default-service-url.com", - type: "livekit", - }, + type: "livekit" + } ], { focus_selection: "oldest_membership", - type: "livekit", + type: "livekit" }, { manageMediaKeys: false, useLegacyMemberEvents: false, useNewMembershipManager: true, - useExperimentalToDeviceTransport: false, - }, + useExperimentalToDeviceTransport: false + } ); }); async function testLeaveRTCSession( cause: "user" | "error", - expectClose: boolean, + expectClose: boolean ): Promise { vi.clearAllMocks(); const session = { leaveRoomSession: vi.fn() } as unknown as MatrixRTCSession; @@ -133,18 +140,18 @@ async function testLeaveRTCSession( expect(session.leaveRoomSession).toHaveBeenCalled(); expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.HangupCall, - expect.anything(), + expect.anything() ); if (expectClose) { expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything(), + expect.anything() ); expect(widget!.api.transport.stop).toHaveBeenCalled(); } else { expect(widget!.api.transport.send).not.toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything(), + expect.anything() ); expect(widget!.api.transport.stop).not.toHaveBeenCalled(); } @@ -172,16 +179,24 @@ test("It fails with configuration error if no live kit url config is set in fall room: { roomId: "roomId", client: { - getDomain: vi.fn().mockReturnValue("example.org"), - }, + getDomain: vi.fn().mockReturnValue("example.org") + } }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), + joinRoomSession: vi.fn() }) as unknown as MatrixRTCSession; - await expect(enterRTCSession(mockedSession, false)).rejects.toThrowError( - expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_FOCUS }), + await expect(enterRTCSession( + mockedSession, + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit" + }, + true + )).rejects.toThrowError( + expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }) ); }); @@ -191,9 +206,9 @@ test("It should not fail with configuration error if homeserver config has livek "org.matrix.msc4143.rtc_foci": [ { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com", - }, - ], + livekit_service_url: "http://my-well-known-service-url.com" + } + ] }); const mockedSession = vi.mocked({ @@ -205,14 +220,19 @@ test("It should not fail with configuration error if homeserver config has livek access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000, - }), - }, + expires_in: 10000 + }) + } }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), + joinRoomSession: vi.fn() }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, false); + await enterRTCSession(mockedSession, { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit" + }, + true); }); diff --git a/src/state/Async.ts b/src/state/Async.ts index 2baa674c..45676759 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -9,12 +9,13 @@ import { catchError, from, map, - Observable, + type Observable, of, - startWith, - switchMap, + startWith } from "rxjs"; +// TODO where are all the comments? ::cry:: +// There used to be an unitialized state!, a state might not start in loading export type Async = | { state: "loading" } | { state: "error"; value: Error } @@ -24,21 +25,22 @@ export const loading: Async = { state: "loading" }; export function error(value: Error): Async { return { state: "error", value }; } + export function ready(value: A): Async { return { state: "ready", value }; } -export function async(promise: Promise): Observable> { +export function async$(promise: Promise): Observable> { return from(promise).pipe( map(ready), startWith(loading), - catchError((e) => of(error(e))), + catchError((e: unknown) => of(error(e as Error ?? new Error("Unknown error")))), ); } export function mapAsync( async: Async, - project: (value: A) => B, + project: (value: A) => B ): Async { return async.state === "ready" ? ready(project(async.value)) : async; } diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 07c78ef6..d9cad2b7 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -68,7 +68,7 @@ import { type ECConnectionState, } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; -import type { RaisedHandInfo } from "../reactions"; +import type { RaisedHandInfo, ReactionInfo } from "../reactions"; import { alice, aliceDoppelganger, @@ -95,6 +95,7 @@ import { ObservableScope } from "./ObservableScope"; import { MediaDevices } from "./MediaDevices"; import { getValue } from "../utils/observable"; import { type Behavior, constant } from "./Behavior"; +import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../UrlParams", () => ({ getUrlParams })); @@ -341,6 +342,7 @@ function withCallViewModel( .mockImplementation((_room, _eventType) => of()); const muteStates = mockMuteStates(); const raisedHands$ = new BehaviorSubject>({}); + const reactions$ = new BehaviorSubject>({}); const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, @@ -349,7 +351,8 @@ function withCallViewModel( muteStates, options, raisedHands$, - new BehaviorSubject({}), + reactions$, + new BehaviorSubject({ processor: undefined, supported: undefined }), ); onTestFinished(() => { diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 2c02521e..4b8ff879 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -132,7 +132,7 @@ import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; import { PublishConnection } from "./PublishConnection.ts"; -import { type Async, async, mapAsync, ready } from "./Async"; +import { type Async, async$, mapAsync, ready } from "./Async"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -520,7 +520,7 @@ export class CallViewModel extends ViewModel { joined ? combineLatest( [ - async(this.preferredTransport), + async$(this.preferredTransport), this.memberships$, multiSfu.value$, ], @@ -1953,7 +1953,10 @@ export class CallViewModel extends ViewModel { .subscribe(({ start, stop }) => { for (const c of stop) { logger.info(`Disconnecting from ${c.localTransport.livekit_service_url}`); - c.stop(); + c.stop().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + });; } for (const c of start) { c.start().then( diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 07a38d7d..74a61515 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, describe, expect, it, type Mock, type MockedObject, vi } from "vitest"; -import type { CallMembership, LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, of } from "rxjs"; import { ConnectionState, @@ -18,8 +17,8 @@ import { import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; -import { type BackgroundOptions, type ProcessorWrapper } from "@livekit/track-processors"; +import type { CallMembership, LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; @@ -29,7 +28,6 @@ import { mockMediaDevices, mockMuteStates } from "../utils/test.ts"; import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; import { type MuteStates } from "./MuteStates.ts"; - let testScope: ObservableScope; let client: MockedObject; @@ -551,7 +549,7 @@ describe("Publishing participants observations", () => { }); - it("should be scoped to parent scope", async () => { + it("should be scoped to parent scope", (): void => { setupTest(); const connection = setupRemoteConnection(); @@ -613,7 +611,7 @@ describe("PublishConnection", () => { let roomFactoryMock: Mock<() => LivekitRoom>; let muteStates: MockedObject; - function setUpPublishConnection() { + function setUpPublishConnection(): void { setupTest(); roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); @@ -673,9 +671,13 @@ describe("PublishConnection", () => { } }; + // TODO understand what is wrong with our mocking that requires ts-expect-error const fakeDevices = mockMediaDevices({ + // @ts-expect-error Mocking only audioInput, + // @ts-expect-error Mocking only videoInput, + // @ts-expect-error Mocking only audioOutput }); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 07bc5665..8a025882 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -88,7 +88,10 @@ class MuteState { } else { subscriber.next(enabled); syncing = true; - sync(); + sync().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + }); } } }; @@ -97,7 +100,10 @@ class MuteState { latestDesired = desired; if (syncing === false) { syncing = true; - sync(); + sync().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + }); } }); return (): void => s.unsubscribe(); @@ -132,6 +138,7 @@ class MuteState { ) {} } +// TODO there is another MuteStates in src/room/MuteStates.tsx ?? why export class MuteStates { public readonly audio = new MuteState( this.scope, diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 672f3334..57be00ef 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -19,7 +19,7 @@ import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; -import { mockLocalParticipant } from "../utils/test"; +import { mockLocalParticipant, mockMatrixRoomMember, mockRtcMembership } from "../utils/test"; describe("MediaView", () => { const participant = mockLocalParticipant({}); @@ -45,7 +45,10 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: undefined, + member: mockMatrixRoomMember( + mockRtcMembership("@alice:example.org", "CCCC"), + { name: "some name" }, + ), localParticipant: false, focusable: true, }; diff --git a/src/utils/test.ts b/src/utils/test.ts index 519fdd50..b77e63c0 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, vi, vitest } from "vitest"; +import { expect, type MockedObject, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -205,6 +205,9 @@ export function mockMatrixRoomMember( return { ...mockEmitter(), userId: rtcMembership.sender, + getMxcAvatarUrl(): string | undefined { + return undefined; + }, ...member, } as RoomMember; } @@ -416,13 +419,13 @@ export const deviceStub = { select(): void {}, }; -export function mockMediaDevices(data: Partial): MediaDevices { - return { +export function mockMediaDevices(data: Partial): MockedObject { + return vi.mocked({ audioInput: deviceStub, audioOutput: deviceStub, videoInput: deviceStub, ...data, - } as MediaDevices; + } as MediaDevices); } export function mockMuteStates( From c820ba39837d04b9c16d41b975bd60bb77bfa310 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:07:46 +0200 Subject: [PATCH 060/144] build: update lock file --- yarn.lock | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4429b7d4..912a13a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5176,6 +5176,13 @@ __metadata: languageName: node linkType: hard +"@types/glob-to-regexp@npm:^0.4.4": + version: 0.4.4 + resolution: "@types/glob-to-regexp@npm:0.4.4" + checksum: 10c0/7288ff853850d8302a8770a3698b187fc3970ad12ee6427f0b3758a3e7a0ebb0bd993abc6ebaaa979d09695b4194157d2bfaa7601b0fb9ed72c688b4c1298b88 + languageName: node + linkType: hard + "@types/grecaptcha@npm:^3.0.9": version: 3.0.9 resolution: "@types/grecaptcha@npm:3.0.9" @@ -7528,6 +7535,7 @@ __metadata: eslint-plugin-react-hooks: "npm:^5.0.0" eslint-plugin-rxjs: "npm:^5.0.3" eslint-plugin-unicorn: "npm:^56.0.0" + fetch-mock: "npm:11.1.5" global-jsdom: "npm:^26.0.0" i18next: "npm:^24.0.0" i18next-browser-languagedetector: "npm:^8.0.0" @@ -8495,6 +8503,22 @@ __metadata: languageName: node linkType: hard +"fetch-mock@npm:11.1.5": + version: 11.1.5 + resolution: "fetch-mock@npm:11.1.5" + dependencies: + "@types/glob-to-regexp": "npm:^0.4.4" + dequal: "npm:^2.0.3" + glob-to-regexp: "npm:^0.4.1" + is-subset: "npm:^0.1.1" + regexparam: "npm:^3.0.0" + peerDependenciesMeta: + node-fetch: + optional: true + checksum: 10c0/f32f1d7879b654a3fab7c3576901193ddd4c63cb9aeae2ed66ff42062400c0937d4696b1a5171e739d5f62470e6554e190f14816789f5e3b2bf1ad90208222e6 + languageName: node + linkType: hard + "fflate@npm:^0.4.8": version: 0.4.8 resolution: "fflate@npm:0.4.8" @@ -8876,6 +8900,13 @@ __metadata: languageName: node linkType: hard +"glob-to-regexp@npm:^0.4.1": + version: 0.4.1 + resolution: "glob-to-regexp@npm:0.4.1" + checksum: 10c0/0486925072d7a916f052842772b61c3e86247f0a80cc0deb9b5a3e8a1a9faad5b04fb6f58986a09f34d3e96cd2a22a24b7e9882fb1cf904c31e9a310de96c429 + languageName: node + linkType: hard + "glob@npm:^10.2.2, glob@npm:^10.3.10, glob@npm:^10.3.7, glob@npm:^10.4.1": version: 10.4.5 resolution: "glob@npm:10.4.5" @@ -9611,6 +9642,13 @@ __metadata: languageName: node linkType: hard +"is-subset@npm:^0.1.1": + version: 0.1.1 + resolution: "is-subset@npm:0.1.1" + checksum: 10c0/d8125598ab9077a76684e18726fb915f5cea7a7358ed0c6ff723f4484d71a0a9981ee5aae06c44de99cfdef0fefce37438c6257ab129e53c82045ea0c2acdebf + languageName: node + linkType: hard + "is-symbol@npm:^1.0.4, is-symbol@npm:^1.1.1": version: 1.1.1 resolution: "is-symbol@npm:1.1.1" @@ -10299,7 +10337,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=ca4a9c655537702daf9a69ed5d94831cebc49666" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4643844597f8bd0196714ecc1c7fafd3f3f6669d" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10353,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/1fb0933d0bb686b0f290b1a62f75eec290b7c52a410d5968c2ccfb527a64e78a58012e1bd8f90c874d385dace3228b9a8c80e114ee227fc8a60e7c9611112ceb + checksum: 10c0/90d6feb7c5214b2fce7b8d6394c88d39a538224dac464e2a5315fb63388126999da28284bc9b7443e035ad0f24c21c1c7d9e1ad4245ee854595e73a390f48c2a languageName: node linkType: hard @@ -12043,6 +12081,13 @@ __metadata: languageName: node linkType: hard +"regexparam@npm:^3.0.0": + version: 3.0.0 + resolution: "regexparam@npm:3.0.0" + checksum: 10c0/a6430d7b97d5a7d5518f37a850b6b73aab479029d02f46af4fa0e8e4a1d7aad05b7a0d2d10c86ded21a14d5f0fa4c68525f873a5fca2efeefcccd93c36627459 + languageName: node + linkType: hard + "regexpu-core@npm:^6.2.0": version: 6.2.0 resolution: "regexpu-core@npm:6.2.0" From 743796119588ea329a5d09150a113308792273a8 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:12:23 +0200 Subject: [PATCH 061/144] lint: fix import order --- src/state/PublishConnection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 6c15fc0f..8381c092 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -13,12 +13,12 @@ import { Track } from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; import type { Behavior } from "./Behavior.ts"; import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; import type { MuteStates } from "./MuteStates.ts"; import { type ProcessorState, trackProcessorSync } from "../livekit/TrackProcessorContext.tsx"; -import { logger } from "../../../matrix-js-sdk/lib/logger"; import { getUrlParams } from "../UrlParams.ts"; import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; From 529cb8a7ec68908f14cbbc5fa509ae0901df4400 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:24:02 +0200 Subject: [PATCH 062/144] prettier ! --- src/livekit/MatrixAudioRenderer.test.tsx | 22 +- src/room/GroupCallView.test.tsx | 2 +- src/room/RoomPage.tsx | 84 +++-- src/rtcSessionHelpers.test.ts | 117 +++--- src/state/Async.ts | 15 +- src/state/CallViewModel.test.ts | 5 +- src/state/CallViewModel.ts | 60 +-- src/state/Connection.test.ts | 443 ++++++++++++----------- src/state/Connection.ts | 131 ++++--- src/state/PublishConnection.ts | 79 ++-- src/tile/MediaView.test.tsx | 6 +- src/utils/test.ts | 4 +- 12 files changed, 547 insertions(+), 421 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index e2464eb8..07592732 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -24,10 +24,9 @@ import { mockMatrixRoomMember, mockMediaDevices, mockRtcMembership, - mockTrack + mockTrack, } from "../utils/test"; - export const TestAudioContextConstructor = vi.fn(() => testAudioContext); const MediaDevicesProvider = MediaDevicesContext.MediaDevicesContext.Provider; @@ -65,8 +64,8 @@ it("should render for member", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -95,8 +94,8 @@ it("should not render without member", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -122,8 +121,8 @@ it("should not setup audioContext gain and pan if there is no need to.", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -160,8 +159,8 @@ it("should setup audioContext gain and pan", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -173,7 +172,8 @@ it("should setup audioContext gain and pan", () => { + livekitRoom={livekitRoom} + /> , ); diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 22d99b31..8c4a276a 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -167,7 +167,7 @@ function createGroupCallView( muteStates={muteState} widget={widget} joined={true} - setJoined={function(value: boolean): void { }} + setJoined={function (value: boolean): void {}} /> diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 3924437b..e9527e03 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -116,20 +116,22 @@ export const RoomPage: FC = () => { const groupCallView = (): ReactNode => { switch (groupCallState.kind) { case "loaded": - return muteStates && ( - + return ( + muteStates && ( + + ) ); case "waitForInvite": case "canKnock": { @@ -148,31 +150,35 @@ export const RoomPage: FC = () => { ); return ( - muteStates && knock?.()} - enterLabel={label} - waitingForInvite={groupCallState.kind === "waitForInvite"} - confineToRoom={confineToRoom} - hideHeader={header !== "standard"} - participantCount={null} - muteStates={muteStates} - onShareClick={null} - /> + muteStates && ( + knock?.()} + enterLabel={label} + waitingForInvite={groupCallState.kind === "waitForInvite"} + confineToRoom={confineToRoom} + hideHeader={header !== "standard"} + participantCount={null} + muteStates={muteStates} + onShareClick={null} + /> + ) ); } case "loading": diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 1058628f..258d2f9a 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -23,38 +23,37 @@ vi.mock("./widget", () => ({ ...actualWidget, widget: { api: { - setAlwaysOnScreen: (): void => { - }, - transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() } + setAlwaysOnScreen: (): void => {}, + transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() }, }, - lazyActions: new EventEmitter() - } + lazyActions: new EventEmitter(), + }, })); test("It joins the correct Session", async () => { const focusFromOlderMembership = { type: "livekit", livekit_service_url: "http://my-oldest-member-service-url.com", - livekit_alias: "my-oldest-member-service-alias" + livekit_alias: "my-oldest-member-service-alias", }; const focusConfigFromWellKnown = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com" + livekit_service_url: "http://my-well-known-service-url.com", }; const focusConfigFromWellKnown2 = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url2.com" + livekit_service_url: "http://my-well-known-service-url2.com", }; const clientWellKnown = { "org.matrix.msc4143.rtc_foci": [ focusConfigFromWellKnown, - focusConfigFromWellKnown2 - ] + focusConfigFromWellKnown2, + ], }; mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" } + livekit: { livekit_service_url: "http://my-default-service-url.com" }, }); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation( @@ -63,7 +62,7 @@ test("It joins the correct Session", async () => { return Promise.resolve(clientWellKnown); } return Promise.resolve({}); - } + }, ); const mockedSession = vi.mocked({ @@ -75,64 +74,67 @@ test("It joins the correct Session", async () => { access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000 - }) - } + expires_in: 10000, + }), + }, }, memberships: [], getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), getOldestMembership: vi.fn().mockReturnValue({ - getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]) + getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), }), - joinRoomSession: vi.fn() + joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, { + await enterRTCSession( + mockedSession, + { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" + type: "livekit", }, - true); + true, + ); expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( [ { livekit_alias: "my-oldest-member-service-alias", livekit_service_url: "http://my-oldest-member-service-url.com", - type: "livekit" + type: "livekit", }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" + type: "livekit", }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url2.com", - type: "livekit" + type: "livekit", }, { livekit_alias: "roomId", livekit_service_url: "http://my-default-service-url.com", - type: "livekit" - } + type: "livekit", + }, ], { focus_selection: "oldest_membership", - type: "livekit" + type: "livekit", }, { manageMediaKeys: false, useLegacyMemberEvents: false, useNewMembershipManager: true, - useExperimentalToDeviceTransport: false - } + useExperimentalToDeviceTransport: false, + }, ); }); async function testLeaveRTCSession( cause: "user" | "error", - expectClose: boolean + expectClose: boolean, ): Promise { vi.clearAllMocks(); const session = { leaveRoomSession: vi.fn() } as unknown as MatrixRTCSession; @@ -140,18 +142,18 @@ async function testLeaveRTCSession( expect(session.leaveRoomSession).toHaveBeenCalled(); expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.HangupCall, - expect.anything() + expect.anything(), ); if (expectClose) { expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything() + expect.anything(), ); expect(widget!.api.transport.stop).toHaveBeenCalled(); } else { expect(widget!.api.transport.send).not.toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything() + expect.anything(), ); expect(widget!.api.transport.stop).not.toHaveBeenCalled(); } @@ -179,24 +181,26 @@ test("It fails with configuration error if no live kit url config is set in fall room: { roomId: "roomId", client: { - getDomain: vi.fn().mockReturnValue("example.org") - } + getDomain: vi.fn().mockReturnValue("example.org"), + }, }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn() + joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await expect(enterRTCSession( - mockedSession, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" - }, - true - )).rejects.toThrowError( - expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }) + await expect( + enterRTCSession( + mockedSession, + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit", + }, + true, + ), + ).rejects.toThrowError( + expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }), ); }); @@ -206,9 +210,9 @@ test("It should not fail with configuration error if homeserver config has livek "org.matrix.msc4143.rtc_foci": [ { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com" - } - ] + livekit_service_url: "http://my-well-known-service-url.com", + }, + ], }); const mockedSession = vi.mocked({ @@ -220,19 +224,22 @@ test("It should not fail with configuration error if homeserver config has livek access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000 - }) - } + expires_in: 10000, + }), + }, }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn() + joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, { + await enterRTCSession( + mockedSession, + { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" + type: "livekit", }, - true); + true, + ); }); diff --git a/src/state/Async.ts b/src/state/Async.ts index 45676759..79de4140 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -5,14 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - catchError, - from, - map, - type Observable, - of, - startWith -} from "rxjs"; +import { catchError, from, map, type Observable, of, startWith } from "rxjs"; // TODO where are all the comments? ::cry:: // There used to be an unitialized state!, a state might not start in loading @@ -34,13 +27,15 @@ export function async$(promise: Promise): Observable> { return from(promise).pipe( map(ready), startWith(loading), - catchError((e: unknown) => of(error(e as Error ?? new Error("Unknown error")))), + catchError((e: unknown) => + of(error((e as Error) ?? new Error("Unknown error"))), + ), ); } export function mapAsync( async: Async, - project: (value: A) => B + project: (value: A) => B, ): Async { return async.state === "ready" ? ready(project(async.value)) : async; } diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index d9cad2b7..acc6a991 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -352,7 +352,10 @@ function withCallViewModel( options, raisedHands$, reactions$, - new BehaviorSubject({ processor: undefined, supported: undefined }), + new BehaviorSubject({ + processor: undefined, + supported: undefined, + }), ); onTestFinished(() => { diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4b8ff879..f517908f 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -126,7 +126,11 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type Connection, type ConnectionOpts, RemoteConnection } from "./Connection"; +import { + type Connection, + type ConnectionOpts, + RemoteConnection, +} from "./Connection"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; @@ -485,7 +489,6 @@ export class CallViewModel extends ViewModel { ), ); - /** * The MatrixRTC session participants. */ @@ -574,7 +577,6 @@ export class CallViewModel extends ViewModel { (transport) => transport && mapAsync(transport, (transport) => { - const opts: ConnectionOpts = { transport, client: this.matrixRTCSession.room.client, @@ -582,15 +584,16 @@ export class CallViewModel extends ViewModel { remoteTransports$: this.remoteTransports$, }; return { - connection: new PublishConnection( - opts, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - transport, - }}), + connection: new PublishConnection( + opts, + this.mediaDevices, + this.muteStates, + this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), + ), + transport, + }; + }), ), ), ); @@ -605,14 +608,14 @@ export class CallViewModel extends ViewModel { this.localConnection$.pipe( switchMap((c) => c?.state === "ready" - // TODO mapping to ConnectionState for compatibility, but we should use the full state? - ? c.value.focusedConnectionState$.pipe( - map((s) => { - if (s.state === "ConnectedToLkRoom") return s.connectionState; - return ConnectionState.Disconnected - }), - distinctUntilChanged(), - ) + ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? + c.value.focusedConnectionState$.pipe( + map((s) => { + if (s.state === "ConnectedToLkRoom") return s.connectionState; + return ConnectionState.Disconnected; + }), + distinctUntilChanged(), + ) : of(ConnectionState.Disconnected), ), ), @@ -659,8 +662,11 @@ export class CallViewModel extends ViewModel { client: this.matrixRTCSession.room.client, scope: this.scope, remoteTransports$: this.remoteTransports$, - } - nextConnection = new RemoteConnection(args, this.e2eeLivekitOptions()); + }; + nextConnection = new RemoteConnection( + args, + this.e2eeLivekitOptions(), + ); } else { logger.log( "SFU remoteConnections$ use prev connection: ", @@ -1952,16 +1958,20 @@ export class CallViewModel extends ViewModel { .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { for (const c of stop) { - logger.info(`Disconnecting from ${c.localTransport.livekit_service_url}`); + logger.info( + `Disconnecting from ${c.localTransport.livekit_service_url}`, + ); c.stop().catch((err) => { // TODO: better error handling logger.error("MuteState: handler error", err); - });; + }); } for (const c of start) { c.start().then( () => - logger.info(`Connected to ${c.localTransport.livekit_service_url}`), + logger.info( + `Connected to ${c.localTransport.livekit_service_url}`, + ), (e) => logger.error( `Failed to start connection to ${c.localTransport.livekit_service_url}`, diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 74a61515..69942270 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -5,21 +5,37 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, describe, expect, it, type Mock, type MockedObject, vi } from "vitest"; +import { + afterEach, + describe, + expect, + it, + type Mock, + type MockedObject, + vi, +} from "vitest"; import { BehaviorSubject, of } from "rxjs"; import { ConnectionState, type LocalParticipant, type RemoteParticipant, type Room as LivekitRoom, - RoomEvent, type RoomOptions + RoomEvent, + type RoomOptions, } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; -import type { CallMembership, LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; +import type { + CallMembership, + LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + type ConnectionOpts, + type FocusConnectionState, + RemoteConnection, +} from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../utils/errors.ts"; @@ -38,28 +54,30 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; let fakeRoomEventEmiter: EventEmitter; -let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; transport: LivekitTransport }[]>; +let fakeMembershipsFocusMap$: BehaviorSubject< + { membership: CallMembership; transport: LivekitTransport }[] +>; const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", - type: "livekit" + type: "livekit", }; function setupTest(): void { testScope = new ObservableScope(); client = vi.mocked({ - getOpenIdToken: vi.fn().mockResolvedValue( - { - "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", - "token_type": "Bearer", - "matrix_server_name": "example.org", - "expires_in": 3600 - } - ), - getDeviceId: vi.fn().mockReturnValue("ABCDEF") + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "rYsmGUEwNjKgJYyeNUkZseJN", + token_type: "Bearer", + matrix_server_name: "example.org", + expires_in: 3600, + }), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), } as unknown as OpenIDClientParts); - fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; transport: LivekitTransport }[]>([]); + fakeMembershipsFocusMap$ = new BehaviorSubject< + { membership: CallMembership; transport: LivekitTransport }[] + >([]); localParticipantEventEmiter = new EventEmitter(); @@ -69,9 +87,15 @@ function setupTest(): void { getTrackPublication: vi.fn().mockReturnValue(undefined), on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter), off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter), - addListener: localParticipantEventEmiter.addListener.bind(localParticipantEventEmiter), - removeListener: localParticipantEventEmiter.removeListener.bind(localParticipantEventEmiter), - removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(localParticipantEventEmiter) + addListener: localParticipantEventEmiter.addListener.bind( + localParticipantEventEmiter, + ), + removeListener: localParticipantEventEmiter.removeListener.bind( + localParticipantEventEmiter, + ), + removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind( + localParticipantEventEmiter, + ), } as unknown as LocalParticipant); fakeRoomEventEmiter = new EventEmitter(); @@ -84,56 +108,45 @@ function setupTest(): void { on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), - removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), - setE2EEEnabled: vi.fn().mockResolvedValue(undefined) + removeListener: + fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + removeAllListeners: + fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + setE2EEEnabled: vi.fn().mockResolvedValue(undefined), } as unknown as LivekitRoom); - } function setupRemoteConnection(): RemoteConnection { - const opts: ConnectionOpts = { client: client, transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN" - } - }; - } - ); + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + return { + status: 200, + body: { + url: "wss://matrix-rtc.m.localhost/livekit/sfu", + jwt: "ATOKEN", + }, + }; + }); - fakeLivekitRoom - .connect - .mockResolvedValue(undefined); + fakeLivekitRoom.connect.mockResolvedValue(undefined); - return new RemoteConnection( - opts, - undefined - ); + return new RemoteConnection(opts, undefined); } - afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); fetchMock.reset(); }); - describe("Start connection states", () => { - it("start in initialized state", () => { setupTest(); @@ -142,15 +155,13 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); - expect(connection.focusedConnectionState$.getValue().state) - .toEqual("Initialized"); + expect(connection.focusedConnectionState$.getValue().state).toEqual( + "Initialized", + ); }); it("fail to getOpenId token then error state", async () => { @@ -162,31 +173,27 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedStates.push(value); }); - const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation(async (): Promise => { - return await deferred.promise; - }); + client.getOpenIdToken.mockImplementation( + async (): Promise => { + return await deferred.promise; + }, + ); - connection.start() - .catch(() => { - // expected to throw - }); + connection.start().catch(() => { + // expected to throw + }); let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); @@ -199,11 +206,14 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState!.state === "FailedToStart") { expect(capturedState!.error.message).toEqual("Something went wrong"); - expect(capturedState!.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(capturedState!.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); } else { - expect.fail("Expected FailedToStart state but got " + capturedState?.state); + expect.fail( + "Expected FailedToStart state but got " + capturedState?.state, + ); } - }); it("fail to get JWT token and error state", async () => { @@ -215,13 +225,10 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { @@ -230,21 +237,17 @@ describe("Start connection states", () => { const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - async () => { - await deferredSFU.promise; - return { - status: 500, - body: "Internal Server Error" - }; - } - ); + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { + await deferredSFU.promise; + return { + status: 500, + body: "Internal Server Error", + }; + }); - - connection.start() - .catch(() => { - // expected to throw - }); + connection.start().catch(() => { + // expected to throw + }); let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); @@ -256,15 +259,19 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState?.state === "FailedToStart") { - expect(capturedState?.error.message).toContain("SFU Config fetch failed with exception Error"); - expect(capturedState?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(capturedState?.error.message).toContain( + "SFU Config fetch failed with exception Error", + ); + expect(capturedState?.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); } else { - expect.fail("Expected FailedToStart state but got " + capturedState?.state); + expect.fail( + "Expected FailedToStart state but got " + capturedState?.state, + ); } - }); - it("fail to connect to livekit error state", async () => { setupTest(); vi.useFakeTimers(); @@ -274,46 +281,36 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedStates.push(value); }); - const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN" - } - }; - } - ); + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + return { + status: 200, + body: { + url: "wss://matrix-rtc.m.localhost/livekit/sfu", + jwt: "ATOKEN", + }, + }; + }); - fakeLivekitRoom - .connect - .mockImplementation(async () => { - await deferredSFU.promise; - throw new Error("Failed to connect to livekit"); - }); + fakeLivekitRoom.connect.mockImplementation(async () => { + await deferredSFU.promise; + throw new Error("Failed to connect to livekit"); + }); - connection.start() - .catch(() => { - // expected to throw - }); + connection.start().catch(() => { + // expected to throw + }); let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); @@ -326,12 +323,17 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState && capturedState?.state === "FailedToStart") { - expect(capturedState.error.message).toContain("Failed to connect to livekit"); - expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(capturedState.error.message).toContain( + "Failed to connect to livekit", + ); + expect(capturedState.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); } else { - expect.fail("Expected FailedToStart state but got " + JSON.stringify(capturedState)); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } - }); it("connection states happy path", async () => { @@ -356,7 +358,6 @@ describe("Start connection states", () => { expect(connectingState?.state).toEqual("ConnectingToLkRoom"); const connectedState = capturedState.shift(); expect(connectedState?.state).toEqual("ConnectedToLkRoom"); - }); it("should relay livekit events once connected", async () => { @@ -378,7 +379,7 @@ describe("Start connection states", () => { ConnectionState.SignalReconnecting, ConnectionState.Connecting, ConnectionState.Connected, - ConnectionState.Reconnecting + ConnectionState.Reconnecting, ]; for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); @@ -387,12 +388,18 @@ describe("Start connection states", () => { for (const state of states) { const s = capturedState.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); - const connectedState = s as FocusConnectionState & { state: "ConnectedToLkRoom" }; + const connectedState = s as FocusConnectionState & { + state: "ConnectedToLkRoom"; + }; expect(connectedState.connectionState).toEqual(state); // should always have the focus info - expect(connectedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); - expect(connectedState.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); + expect(connectedState.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); + expect(connectedState.focus.livekit_service_url).toEqual( + livekitFocus.livekit_service_url, + ); } // If the state is not ConnectedToLkRoom, no events should be relayed anymore @@ -403,10 +410,8 @@ describe("Start connection states", () => { } expect(capturedState.length).toEqual(0); - }); - it("shutting down the scope should stop the connection", async () => { setupTest(); vi.useFakeTimers(); @@ -423,7 +428,6 @@ describe("Start connection states", () => { const stopSpy = vi.spyOn(connection, "stop"); testScope.end(); - expect(stopSpy).toHaveBeenCalled(); expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); @@ -437,26 +441,22 @@ describe("Start connection states", () => { expect(capturedState.length).toEqual(0); }); - }); - function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { return vi.mocked({ - identity: id + identity: id, } as unknown as RemoteParticipant); } function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { return vi.mocked({ sender: userId, - deviceId: deviceId + deviceId: deviceId, } as unknown as CallMembership); } describe("Publishing participants observations", () => { - - it("should emit the list of publishing participants", async () => { setupTest(); @@ -464,13 +464,24 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); - const observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + const observedPublishers: { + participant: RemoteParticipant; + membership: CallMembership; + }[][] = []; connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); - if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { + if ( + publishers.some( + (p) => p.participant.identity === "@bob:example.org:DEV111", + ) + ) { bobIsAPublisher.resolve(); } - if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { + if ( + publishers.some( + (p) => p.participant.identity === "@dan:example.org:DEV333", + ) + ) { danIsAPublisher.resolve(); } }); @@ -482,14 +493,13 @@ describe("Publishing participants observations", () => { fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333") + fakeRemoteLivekitParticipant("@dan:example.org:DEV333"), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); for (const participant of participants) { fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); @@ -498,20 +508,27 @@ describe("Publishing participants observations", () => { // At this point there should be no publishers expect(observedPublishers.pop()!.length).toEqual(0); - const otherFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", - type: "livekit" + type: "livekit", }; - const rtcMemberships = [ // Say bob is on the same focus - { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), transport: livekitFocus }, + { + membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), + transport: livekitFocus, + }, // Alice and carol is on a different focus - { membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), transport: otherFocus }, - { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), transport: otherFocus } + { + membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), + transport: otherFocus, + }, + { + membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), + transport: otherFocus, + }, // NO DAVE YET ]; // signal this change in rtc memberships @@ -521,53 +538,74 @@ describe("Publishing participants observations", () => { await bobIsAPublisher.promise; const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + expect(publishers?.[0].participant.identity).toEqual( + "@bob:example.org:DEV111", + ); // Now let's make dan join the rtc memberships - rtcMemberships - .push({ membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), transport: livekitFocus }); + rtcMemberships.push({ + membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), + transport: livekitFocus, + }); fakeMembershipsFocusMap$.next(rtcMemberships); // We should have bob and dan has publishers now await danIsAPublisher.promise; const twoPublishers = observedPublishers.pop(); expect(twoPublishers?.length).toEqual(2); - expect(twoPublishers?.some((p) => p.participant.identity === "@bob:example.org:DEV111")).toBeTruthy(); - expect(twoPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + expect( + twoPublishers?.some( + (p) => p.participant.identity === "@bob:example.org:DEV111", + ), + ).toBeTruthy(); + expect( + twoPublishers?.some( + (p) => p.participant.identity === "@dan:example.org:DEV333", + ), + ).toBeTruthy(); // Now let's make bob leave the livekit room - participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); - fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + participants = participants.filter( + (p) => p.identity !== "@bob:example.org:DEV111", + ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); + fakeRoomEventEmiter.emit( + RoomEvent.ParticipantDisconnected, + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + ); const updatedPublishers = observedPublishers.pop(); expect(updatedPublishers?.length).toEqual(1); - expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + expect( + updatedPublishers?.some( + (p) => p.participant.identity === "@dan:example.org:DEV333", + ), + ).toBeTruthy(); }); - it("should be scoped to parent scope", (): void => { setupTest(); const connection = setupRemoteConnection(); - let observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + let observedPublishers: { + participant: RemoteParticipant; + membership: CallMembership; + }[][] = []; connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); }); let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111") + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); for (const participant of participants) { fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); @@ -578,7 +616,10 @@ describe("Publishing participants observations", () => { const rtcMemberships = [ // Say bob is on the same focus - { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), transport: livekitFocus } + { + membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), + transport: livekitFocus, + }, ]; // signal this change in rtc memberships fakeMembershipsFocusMap$.next(rtcMemberships); @@ -586,27 +627,31 @@ describe("Publishing participants observations", () => { // We should have bob has a publisher now const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + expect(publishers?.[0].participant.identity).toEqual( + "@bob:example.org:DEV111", + ); // end the parent scope testScope.end(); observedPublishers = []; // SHOULD NOT emit any more publishers as the scope is ended - participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); - fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + participants = participants.filter( + (p) => p.identity !== "@bob:example.org:DEV111", + ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); + fakeRoomEventEmiter.emit( + RoomEvent.ParticipantDisconnected, + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + ); expect(observedPublishers.length).toEqual(0); }); }); - describe("PublishConnection", () => { - // let fakeBlurProcessor: ProcessorWrapper; let roomFactoryMock: Mock<() => LivekitRoom>; let muteStates: MockedObject; @@ -616,7 +661,6 @@ describe("PublishConnection", () => { roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); - muteStates = mockMuteStates(); // fakeBlurProcessor = vi.mocked>({ @@ -626,20 +670,15 @@ describe("PublishConnection", () => { // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), // isRunning: vi.fn().mockReturnValue(false) // }); - - } - describe("Livekit room creation", () => { - - function createSetup(): void { setUpPublishConnection(); const fakeTrackProcessorSubject$ = new BehaviorSubject({ supported: true, - processor: undefined + processor: undefined, }); const opts: ConnectionOpts = { @@ -647,28 +686,25 @@ describe("PublishConnection", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: roomFactoryMock + livekitRoomFactory: roomFactoryMock, }; const audioInput = { available$: of(new Map([["mic1", { id: "mic1" }]])), selected$: new BehaviorSubject({ id: "mic1" }), - select(): void { - } + select(): void {}, }; const videoInput = { available$: of(new Map([["cam1", { id: "cam1" }]])), selected$: new BehaviorSubject({ id: "cam1" }), - select(): void { - } + select(): void {}, }; const audioOutput = { available$: of(new Map([["speaker", { id: "speaker" }]])), selected$: new BehaviorSubject({ id: "speaker" }), - select(): void { - } + select(): void {}, }; // TODO understand what is wrong with our mocking that requires ts-expect-error @@ -678,7 +714,7 @@ describe("PublishConnection", () => { // @ts-expect-error Mocking only videoInput, // @ts-expect-error Mocking only - audioOutput + audioOutput, }); new PublishConnection( @@ -686,18 +722,17 @@ describe("PublishConnection", () => { fakeDevices, muteStates, undefined, - fakeTrackProcessorSubject$ + fakeTrackProcessorSubject$, ); - } it("should create room with proper initial audio and video settings", () => { - createSetup(); expect(roomFactoryMock).toHaveBeenCalled(); - const lastCallArgs = roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; + const lastCallArgs = + roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; expect(roomOptions).toBeDefined(); @@ -705,7 +740,6 @@ describe("PublishConnection", () => { expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); - }); it("respect controlledAudioDevices", () => { @@ -719,7 +753,6 @@ describe("PublishConnection", () => { // }) // }; // }); - }); }); }); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 42423938..e5e108b7 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -5,12 +5,27 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; -import { type ConnectionState, type E2EEOptions, Room as LivekitRoom, type RoomOptions } from "livekit-client"; -import { type CallMembership, type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + connectedParticipantsObserver, + connectionStateObserver, +} from "@livekit/components-core"; +import { + type ConnectionState, + type E2EEOptions, + Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { + type CallMembership, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest } from "rxjs"; -import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU"; +import { + getSFUConfigWithOpenID, + type OpenIDClientParts, + type SFUConfig, +} from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; @@ -23,20 +38,26 @@ export interface ConnectionOpts { /** The observable scope to use for this connection. */ scope: ObservableScope; /** An observable of the current RTC call memberships and their associated focus. */ - remoteTransports$: Behavior<{ membership: CallMembership; transport: LivekitTransport }[]>; + remoteTransports$: Behavior< + { membership: CallMembership; transport: LivekitTransport }[] + >; /** Optional factory to create the Livekit room, mainly for testing purposes. */ livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; } export type FocusConnectionState = - | { state: 'Initialized' } - | { state: 'FetchingConfig', focus: LivekitTransport } - | { state: 'ConnectingToLkRoom', focus: LivekitTransport } - | { state: 'PublishingTracks', focus: LivekitTransport } - | { state: 'FailedToStart', error: Error, focus: LivekitTransport } - | { state: 'ConnectedToLkRoom', connectionState: ConnectionState, focus: LivekitTransport } - | { state: 'Stopped', focus: LivekitTransport }; + | { state: "Initialized" } + | { state: "FetchingConfig"; focus: LivekitTransport } + | { state: "ConnectingToLkRoom"; focus: LivekitTransport } + | { state: "PublishingTracks"; focus: LivekitTransport } + | { state: "FailedToStart"; error: Error; focus: LivekitTransport } + | { + state: "ConnectedToLkRoom"; + connectionState: ConnectionState; + focus: LivekitTransport; + } + | { state: "Stopped"; focus: LivekitTransport }; /** * A connection to a Matrix RTC LiveKit backend. @@ -44,10 +65,9 @@ export type FocusConnectionState = * Expose observables for participants and connection state. */ export class Connection { - // Private Behavior - private readonly _focusedConnectionState$ - = new BehaviorSubject({ state: 'Initialized' }); + private readonly _focusedConnectionState$ = + new BehaviorSubject({ state: "Initialized" }); /** * The current state of the connection to the focus server. @@ -71,31 +91,44 @@ export class Connection { public async start(): Promise { this.stopped = false; try { - this._focusedConnectionState$.next({ state: 'FetchingConfig', focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "FetchingConfig", + focus: this.localTransport, + }); // TODO could this be loaded earlier to save time? const { url, jwt } = await this.getSFUConfigWithOpenID(); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._focusedConnectionState$.next({ state: 'ConnectingToLkRoom', focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "ConnectingToLkRoom", + focus: this.localTransport, + }); await this.livekitRoom.connect(url, jwt); // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', focus: this.localTransport, connectionState: this.livekitRoom.state }); + this._focusedConnectionState$.next({ + state: "ConnectedToLkRoom", + focus: this.localTransport, + connectionState: this.livekitRoom.state, + }); } catch (error) { - this._focusedConnectionState$.next({ state: 'FailedToStart', error: error instanceof Error ? error : new Error(`${error}`), focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "FailedToStart", + error: error instanceof Error ? error : new Error(`${error}`), + focus: this.localTransport, + }); throw error; } } - protected async getSFUConfigWithOpenID(): Promise { return await getSFUConfigWithOpenID( this.client, this.localTransport.livekit_service_url, - this.localTransport.livekit_alias - ) + this.localTransport.livekit_alias, + ); } /** * Stops the connection. @@ -106,11 +139,13 @@ export class Connection { public async stop(): Promise { if (this.stopped) return; await this.livekitRoom.disconnect(); - this._focusedConnectionState$.next({ state: 'Stopped', focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "Stopped", + focus: this.localTransport, + }); this.stopped = true; } - /** * An observable of the participants that are publishing on this connection. * This is derived from `participantsIncludingSubscribers$` and `membershipsFocusMap$`. @@ -135,20 +170,20 @@ export class Connection { public readonly livekitRoom: LivekitRoom, opts: ConnectionOpts, ) { - const { transport, client, scope, remoteTransports$ } = - opts; + const { transport, client, scope, remoteTransports$ } = opts; - this.livekitRoom = livekitRoom + this.livekitRoom = livekitRoom; this.localTransport = transport; this.client = client; this.focusedConnectionState$ = scope.behavior( - this._focusedConnectionState$, { state: 'Initialized' } + this._focusedConnectionState$, + { state: "Initialized" }, ); const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), - [] + [], ); this.publishingParticipants$ = scope.behavior( @@ -161,7 +196,7 @@ export class Connection { transport.livekit_service_url === this.localTransport.livekit_service_url ? [membership] - : [] + : [], ) // Pair with their associated LiveKit participant (if any) // Uses flatMap to filter out memberships with no associated rtc participant ([]) @@ -171,18 +206,22 @@ export class Connection { return participant ? [{ participant, membership }] : []; }), ), - [] + [], ); - scope.behavior( - connectionStateObserver(this.livekitRoom) - ).subscribe((connectionState) => { - const current = this._focusedConnectionState$.value; - // Only update the state if we are already connected to the LiveKit room. - if (current.state === 'ConnectedToLkRoom') { - this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); - } - }); + scope + .behavior(connectionStateObserver(this.livekitRoom)) + .subscribe((connectionState) => { + const current = this._focusedConnectionState$.value; + // Only update the state if we are already connected to the LiveKit room. + if (current.state === "ConnectedToLkRoom") { + this._focusedConnectionState$.next({ + state: "ConnectedToLkRoom", + connectionState, + focus: current.focus, + }); + } + }); scope.onEnd(() => void this.stop()); } @@ -195,17 +234,21 @@ export class Connection { * It does not publish any local tracks. */ export class RemoteConnection extends Connection { - /** * Creates a new remote connection to a matrix RTC LiveKit backend. * @param opts * @param sharedE2eeOption - The shared E2EE options to use for the connection. */ - public constructor(opts: ConnectionOpts, sharedE2eeOption: E2EEOptions | undefined) { - const factory = opts.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + public constructor( + opts: ConnectionOpts, + sharedE2eeOption: E2EEOptions | undefined, + ) { + const factory = + opts.livekitRoomFactory ?? + ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); const livekitRoom = factory({ ...defaultLiveKitOptions, - e2ee: sharedE2eeOption + e2ee: sharedE2eeOption, }); super(livekitRoom, opts); } diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 8381c092..c35c71e4 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -10,15 +10,24 @@ import { LocalVideoTrack, Room as LivekitRoom, type RoomOptions, - Track + Track, } from "livekit-client"; -import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; +import { + map, + NEVER, + type Observable, + type Subscription, + switchMap, +} from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import type { Behavior } from "./Behavior.ts"; import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; import type { MuteStates } from "./MuteStates.ts"; -import { type ProcessorState, trackProcessorSync } from "../livekit/TrackProcessorContext.tsx"; +import { + type ProcessorState, + trackProcessorSync, +} from "../livekit/TrackProcessorContext.tsx"; import { getUrlParams } from "../UrlParams.ts"; import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; @@ -31,7 +40,6 @@ import { type ObservableScope } from "./ObservableScope.ts"; * This connection will publish the local user's audio and video tracks. */ export class PublishConnection extends Connection { - /** * Creates a new PublishConnection. * @param args - The connection options. {@link ConnectionOpts} @@ -45,15 +53,22 @@ export class PublishConnection extends Connection { devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, - trackerProcessorState$: Behavior + trackerProcessorState$: Behavior, ) { const { scope } = args; logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); - const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const factory = + args.livekitRoomFactory ?? + ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); const room = factory( - generateRoomOption(devices, trackerProcessorState$.value, controlledAudioDevices, e2eeLivekitOptions) + generateRoomOption( + devices, + trackerProcessorState$.value, + controlledAudioDevices, + e2eeLivekitOptions, + ), ); room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { logger.error("Failed to set E2EE enabled on room", e); @@ -83,14 +98,14 @@ export class PublishConnection extends Connection { public async start(): Promise { this.stopped = false; - await super.start() + await super.start(); if (this.stopped) return; // TODO this can throw errors? It will also prompt for permissions if not already granted const tracks = await this.livekitRoom.localParticipant.createTracks({ audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value + video: this.muteStates.video.enabled$.value, }); if (this.stopped) return; for (const track of tracks) { @@ -100,7 +115,7 @@ export class PublishConnection extends Connection { if (this.stopped) return; // TODO: check if the connection is still active? and break the loop if not? } - }; + } /// Private methods @@ -112,17 +127,19 @@ export class PublishConnection extends Connection { // in the LocalParticipant object for the track object and there's not a nice // way to do that generically. There is usually no OS-level default video capture // device anyway, and audio outputs work differently. - private workaroundRestartAudioInputTrackChrome(devices: MediaDevices, scope: ObservableScope): void { - + private workaroundRestartAudioInputTrackChrome( + devices: MediaDevices, + scope: ObservableScope, + ): void { devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - scope.bind() + scope.bind(), ) .subscribe(() => { if (this.livekitRoom.state != ConnectionState.Connected) return; const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values() + this.livekitRoom.localParticipant.audioTrackPublications.values(), ).find((d) => d.source === Track.Source.Microphone)?.track; if ( @@ -147,11 +164,15 @@ export class PublishConnection extends Connection { }); } -// Observe changes in the selected media devices and update the LiveKit room accordingly. - private observeMediaDevices(scope: ObservableScope, devices: MediaDevices, controlledAudioDevices: boolean):void { + // Observe changes in the selected media devices and update the LiveKit room accordingly. + private observeMediaDevices( + scope: ObservableScope, + devices: MediaDevices, + controlledAudioDevices: boolean, + ): void { const syncDevice = ( kind: MediaDeviceKind, - selected$: Observable + selected$: Observable, ): Subscription => selected$.pipe(scope.bind()).subscribe((device) => { if (this.livekitRoom.state != ConnectionState.Connected) return; @@ -160,7 +181,7 @@ export class PublishConnection extends Connection { "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", this.livekitRoom.getActiveDevice(kind), " !== ", - device?.id + device?.id, ); if ( device !== undefined && @@ -169,7 +190,7 @@ export class PublishConnection extends Connection { this.livekitRoom .switchActiveDevice(kind, device.id) .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e) + logger.error(`Failed to sync ${kind} device with LiveKit`, e), ); } }); @@ -208,21 +229,23 @@ export class PublishConnection extends Connection { }); } - private observeTrackProcessors(scope: ObservableScope, room: LivekitRoom, trackerProcessorState$: Behavior): void { + private observeTrackProcessors( + scope: ObservableScope, + room: LivekitRoom, + trackerProcessorState$: Behavior, + ): void { const track$ = scope.behavior( observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( map((trackRef) => { const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; - }) - ) + }), + ), ); trackProcessorSync(track$, trackerProcessorState$); } - } - // Generate the initial LiveKit RoomOptions based on the current media devices and processor state. function generateRoomOption( devices: MediaDevices, @@ -235,11 +258,11 @@ function generateRoomOption( videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, deviceId: devices.videoInput.selected$.value?.id, - processor: processorState.processor + processor: processorState.processor, }, audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id + deviceId: devices.audioInput.selected$.value?.id, }, audioOutput: { // When using controlled audio devices, we don't want to set the @@ -247,8 +270,8 @@ function generateRoomOption( // (also the id does not need to match a browser device id) deviceId: controlledAudioDevices ? undefined - : getValue(devices.audioOutput.selected$)?.id + : getValue(devices.audioOutput.selected$)?.id, }, - e2ee: e2eeLivekitOptions + e2ee: e2eeLivekitOptions, }; } diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 57be00ef..3637e8de 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -19,7 +19,11 @@ import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; -import { mockLocalParticipant, mockMatrixRoomMember, mockRtcMembership } from "../utils/test"; +import { + mockLocalParticipant, + mockMatrixRoomMember, + mockRtcMembership, +} from "../utils/test"; describe("MediaView", () => { const participant = mockLocalParticipant({}); diff --git a/src/utils/test.ts b/src/utils/test.ts index b77e63c0..98a2addf 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -419,7 +419,9 @@ export const deviceStub = { select(): void {}, }; -export function mockMediaDevices(data: Partial): MockedObject { +export function mockMediaDevices( + data: Partial, +): MockedObject { return vi.mocked({ audioInput: deviceStub, audioOutput: deviceStub, From 18ba02c9c26d78d9fb0cd9c67834bccccb78e089 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:29:11 +0200 Subject: [PATCH 063/144] knip: remove dead code --- src/room/MuteStates.ts | 11 ++++---- src/useMatrixRTCSessionJoinState.ts | 40 ----------------------------- src/utils/test-fixtures.ts | 3 +-- 3 files changed, 7 insertions(+), 47 deletions(-) delete mode 100644 src/useMatrixRTCSessionJoinState.ts diff --git a/src/room/MuteStates.ts b/src/room/MuteStates.ts index e89d13d9..dfc599e7 100644 --- a/src/room/MuteStates.ts +++ b/src/room/MuteStates.ts @@ -27,11 +27,12 @@ import { ElementWidgetActions, widget } from "../widget"; import { Config } from "../config/Config"; import { useUrlParams } from "../UrlParams"; -/** - * If there already are this many participants in the call, we automatically mute - * the user. - */ -export const MUTE_PARTICIPANT_COUNT = 8; +// /** +// * If there already are this many participants in the call, we automatically mute +// * the user. +// */ +// TODO: multi-sfu dead code? +// export const MUTE_PARTICIPANT_COUNT = 8; interface DeviceAvailable { enabled: boolean; diff --git a/src/useMatrixRTCSessionJoinState.ts b/src/useMatrixRTCSessionJoinState.ts deleted file mode 100644 index 2f6ccf25..00000000 --- a/src/useMatrixRTCSessionJoinState.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { logger } from "matrix-js-sdk/lib/logger"; -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, -} from "matrix-js-sdk/lib/matrixrtc"; -import { TypedEventEmitter } from "matrix-js-sdk"; -import { useCallback, useEffect } from "react"; - -import { useTypedEventEmitterState } from "./useEvents"; - -const dummySession = new TypedEventEmitter(); - -export function useMatrixRTCSessionJoinState( - rtcSession: MatrixRTCSession | undefined, -): boolean { - // React doesn't allow you to run a hook conditionally, so we have to plug in - // a dummy event emitter in case there is no rtcSession yet - const isJoined = useTypedEventEmitterState( - rtcSession ?? dummySession, - MatrixRTCSessionEvent.JoinStateChanged, - useCallback(() => rtcSession?.isJoined() ?? false, [rtcSession]), - ); - - useEffect(() => { - logger.info( - `Session in room ${rtcSession?.room.roomId} changed to ${ - isJoined ? "joined" : "left" - }`, - ); - }, [rtcSession, isJoined]); - - return isJoined; -} diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index 6a8b641b..9d93267e 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -9,7 +9,6 @@ import { mockRtcMembership, mockMatrixRoomMember, mockRemoteParticipant, - mockLocalParticipant, } from "./test"; export const localRtcMember = mockRtcMembership("@carol:example.org", "1111"); @@ -18,7 +17,7 @@ export const localRtcMemberDevice2 = mockRtcMembership( "2222", ); export const local = mockMatrixRoomMember(localRtcMember); -export const localParticipant = mockLocalParticipant({ identity: "" }); +// export const localParticipant = mockLocalParticipant({ identity: "" }); export const localId = `${local.userId}:${localRtcMember.deviceId}`; export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); From 05e7b5a7ffb71a7a916b25632b526a8bf3a8f397 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 17:35:25 +0200 Subject: [PATCH 064/144] fixup MediaView tests --- src/tile/MediaView.test.tsx | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 3637e8de..672f3334 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -19,11 +19,7 @@ import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; -import { - mockLocalParticipant, - mockMatrixRoomMember, - mockRtcMembership, -} from "../utils/test"; +import { mockLocalParticipant } from "../utils/test"; describe("MediaView", () => { const participant = mockLocalParticipant({}); @@ -49,10 +45,7 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: mockMatrixRoomMember( - mockRtcMembership("@alice:example.org", "CCCC"), - { name: "some name" }, - ), + member: undefined, localParticipant: false, focusable: true, }; From 669bc76dd543cba3658684a953ae2b77e6e11e3f Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 01:04:58 -0400 Subject: [PATCH 065/144] Replace calls to deprecated resolveActiveFocus --- src/state/CallViewModel.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 1a20589c..5e037b05 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -523,10 +523,12 @@ export class CallViewModel extends ViewModel { multiSfu.value$, ], (preferred, memberships, multiSfu) => { + const oldestMembership = + this.matrixRTCSession.getOldestMembership(); const remote = memberships.flatMap((m) => { if (m.sender === this.userId && m.deviceId === this.deviceId) return []; - const t = this.matrixRTCSession.resolveActiveFocus(m); + const t = m.getTransport(oldestMembership ?? m); return t && isLivekitTransport(t) ? [{ membership: m, transport: t }] : []; @@ -617,10 +619,11 @@ export class CallViewModel extends ViewModel { // Until the local transport becomes ready we have no idea which // transports will actually need a dedicated remote connection if (transports?.local.state === "ready") { + const oldestMembership = this.matrixRTCSession.getOldestMembership(); const localServiceUrl = transports.local.value.livekit_service_url; const remoteServiceUrls = new Set( transports.remote.flatMap(({ membership, transport }) => { - const t = this.matrixRTCSession.resolveActiveFocus(membership); + const t = membership.getTransport(oldestMembership ?? membership); return t && isLivekitTransport(t) && t.livekit_service_url !== localServiceUrl From 13fb46644c48735fcf69837439f09bcfbd346210 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 08:50:35 +0200 Subject: [PATCH 066/144] test: Fix mediaView test, ,member is not optional anymore Updated the test because now name will be the userId instead of default display name --- src/tile/MediaView.test.tsx | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 672f3334..abf29313 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it, test } from "vitest"; +import { describe, expect, it, test, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { TooltipProvider } from "@vector-im/compound-web"; @@ -16,6 +16,7 @@ import { import { LocalTrackPublication, Track } from "livekit-client"; import { TrackInfo } from "@livekit/protocol"; import { type ComponentProps } from "react"; +import { type RoomMember } from "matrix-js-sdk"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; @@ -45,7 +46,11 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: undefined, + member: vi.mocked({ + name: () => "some name", + userId: "@alice:example.com", + getMxcAvatarUrl: vi.fn().mockReturnValue(undefined), + }), localParticipant: false, focusable: true, }; @@ -59,9 +64,9 @@ describe("MediaView", () => { test("neither video nor avatar are shown", () => { render(); expect(screen.queryByTestId("video")).toBeNull(); - expect(screen.queryAllByRole("img", { name: "some name" }).length).toBe( - 0, - ); + expect( + screen.queryAllByRole("img", { name: "@alice:example.com" }).length, + ).toBe(0); }); }); @@ -70,14 +75,18 @@ describe("MediaView", () => { render( , ); - expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect( + screen.getByRole("img", { name: "@alice:example.com" }), + ).toBeVisible(); expect(screen.queryAllByText("Waiting for media...").length).toBe(0); }); it("shows avatar and label for remote user", () => { render( , ); - expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect( + screen.getByRole("img", { name: "@alice:example.com" }), + ).toBeVisible(); expect(screen.getByText("Waiting for media...")).toBeVisible(); }); }); @@ -131,7 +140,9 @@ describe("MediaView", () => { , ); - expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect( + screen.getByRole("img", { name: "@alice:example.com" }), + ).toBeVisible(); expect(screen.getByTestId("video")).not.toBeVisible(); }); }); From f5ea734a5c3dea09f5c7e4cd31339745823b9b48 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 14:29:59 +0200 Subject: [PATCH 067/144] esLint fix --- src/tile/MediaView.test.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index abf29313..c26a4d5f 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -47,10 +47,9 @@ describe("MediaView", () => { unencryptedWarning: false, video: trackReference, member: vi.mocked({ - name: () => "some name", userId: "@alice:example.com", getMxcAvatarUrl: vi.fn().mockReturnValue(undefined), - }), + } as unknown as RoomMember), localParticipant: false, focusable: true, }; From afe004c6e7306582e6eacd3ec7e1c8c96207d9c4 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 14:30:52 +0200 Subject: [PATCH 068/144] Remove un-necessary transport field, already accessible from connection --- src/state/CallViewModel.ts | 50 ++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index f517908f..e0045d15 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -563,7 +563,7 @@ export class CallViewModel extends ViewModel { /** * The transport over which we should be actively publishing our media. */ - private readonly localTransport$: Behavior | null> = + private readonly localTransport$: Behavior> = this.scope.behavior( this.transports$.pipe( map((transports) => transports?.local ?? null), @@ -571,38 +571,30 @@ export class CallViewModel extends ViewModel { ), ); - private readonly localConnectionAndTransport$ = this.scope.behavior( - this.localTransport$.pipe( - map( - (transport) => - transport && - mapAsync(transport, (transport) => { - const opts: ConnectionOpts = { - transport, - client: this.matrixRTCSession.room.client, - scope: this.scope, - remoteTransports$: this.remoteTransports$, - }; - return { - connection: new PublishConnection( + private readonly localConnection$: Behavior> = + this.scope.behavior( + this.localTransport$.pipe( + map( + (transport) => + transport && + mapAsync(transport, (transport) => { + const opts: ConnectionOpts = { + transport, + client: this.matrixRTCSession.room.client, + scope: this.scope, + remoteTransports$: this.remoteTransports$, + }; + return new PublishConnection( opts, this.mediaDevices, this.muteStates, this.e2eeLivekitOptions(), this.scope.behavior(this.trackProcessorState$), - ), - transport, - }; - }), + ); + }), + ), ), - ), - ); - - private readonly localConnection$ = this.scope.behavior( - this.localConnectionAndTransport$.pipe( - map((value) => value && mapAsync(value, ({ connection }) => connection)), - ), - ); + ); public readonly livekitConnectionState$ = this.scope.behavior( this.localConnection$.pipe( @@ -813,11 +805,11 @@ export class CallViewModel extends ViewModel { }[] >( // TODO: Move this logic into Connection/PublishConnection if possible - this.localConnectionAndTransport$ + this.localConnection$ .pipe( switchMap((values) => { if (values?.state !== "ready") return []; - const localConnection = values.value.connection; + const localConnection = values.value; const memberError = (): never => { throw new Error("No room member for call membership"); }; From 427a8dd644dd86271b8122c763c29e0a71cf08e2 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 14:48:40 +0200 Subject: [PATCH 069/144] test: Fix Audio render tests and added more --- src/livekit/MatrixAudioRenderer.test.tsx | 223 ++++++++++++++--------- src/livekit/MatrixAudioRenderer.tsx | 18 +- src/utils/test.ts | 7 +- 3 files changed, 148 insertions(+), 100 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 07592732..c1ee6f83 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -6,21 +6,24 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { render } from "@testing-library/react"; +import { render, type RenderResult } from "@testing-library/react"; import { getTrackReferenceId, type TrackReference, } from "@livekit/components-core"; -import { type RemoteAudioTrack } from "livekit-client"; +import { + type Participant, + type RemoteAudioTrack, + type RemoteParticipant, + type Room, +} from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; -import { of } from "rxjs"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; import { - mockLivekitRoom, mockMatrixRoomMember, mockMediaDevices, mockRtcMembership, @@ -54,90 +57,148 @@ vi.mock("@livekit/components-react", async (importOriginal) => { }; }); -const tracks = [mockTrack("test:123")]; -vi.mocked(useTracks).mockReturnValue(tracks); +let tracks: TrackReference[] = []; -it("should render for member", () => { - // TODO this is duplicated test setup in all tests - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - const { container, queryAllByTestId } = render( +/** + * Render the test component with given rtc members and livekit participant identities. + * + * It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa. + * + * @param rtcMembers - Array of active rtc members with userId and deviceId. + * @param livekitParticipantIdentities - Array of livekit participant (that are publishing). + * */ + +function renderTestComponent( + rtcMembers: { userId: string; deviceId: string }[], + livekitParticipantIdentities: ({ id: string; isLocal?: boolean } | string)[], +): RenderResult { + const liveKitParticipants = livekitParticipantIdentities.map((p) => { + const identity = typeof p === "string" ? p : p.id; + const isLocal = typeof p === "string" ? false : (p.isLocal ?? false); + return vi.mocked({ + identity, + isLocal, + } as unknown as RemoteParticipant); + }); + const participants = rtcMembers.map(({ userId, deviceId }) => { + const p = liveKitParticipants.find( + (p) => p.identity === `${userId}:${deviceId}`, + ); + const localRtcMember = mockRtcMembership(userId, deviceId); + const member = mockMatrixRoomMember(localRtcMember); + return { + id: `${userId}:${deviceId}`, + participant: p, + member, + }; + }); + const livekitRoom = vi.mocked({ + remoteParticipants: new Map( + liveKitParticipants.map((p) => [p.identity, p]), + ), + } as unknown as Room); + + tracks = participants + .filter((p) => p.participant) + .map((p) => mockTrack(p.participant!)) as TrackReference[]; + + vi.mocked(useTracks).mockReturnValue(tracks); + return render( , ); +} + +it("should render for member", () => { + const { container, queryAllByTestId } = renderTestComponent( + [{ userId: "@alice", deviceId: "DEV0" }], + ["@alice:DEV0"], + ); expect(container).toBeTruthy(); expect(queryAllByTestId("audio")).toHaveLength(1); }); it("should not render without member", () => { - // const memberships = [ - // { sender: "othermember", deviceId: "123" }, - // ] as CallMembership[]; - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - const { container, queryAllByTestId } = render( - - - , + const { container, queryAllByTestId } = renderTestComponent( + [{ userId: "@bob", deviceId: "DEV0" }], + ["@alice:DEV0"], ); expect(container).toBeTruthy(); expect(queryAllByTestId("audio")).toHaveLength(0); }); +const TEST_CASES: { + rtcUsers: { userId: string; deviceId: string }[]; + livekitParticipantIdentities: (string | { id: string; isLocal?: boolean })[]; + expectedAudioTracks: number; +}[] = [ + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@alice", deviceId: "DEV1" }, + { userId: "@bob", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: [ + { id: "@alice:DEV0" }, + "@bob:DEV0", + "@alice:DEV1", + ], + expectedAudioTracks: 3, + }, + // Alice DEV0 is local participant, should not render + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@alice", deviceId: "DEV1" }, + { userId: "@bob", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: [ + { id: "@alice:DEV0", isLocal: true }, + "@bob:DEV0", + "@alice:DEV1", + ], + expectedAudioTracks: 2, + }, + // Charlie is a rtc member but not in livekit + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@bob", deviceId: "DEV0" }, + { userId: "@charlie", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: ["@alice:DEV0", { id: "@bob:DEV0" }], + expectedAudioTracks: 2, + }, + // Charlie is in livekit but not rtc member + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@bob", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"], + expectedAudioTracks: 2, + }, +]; + +TEST_CASES.forEach( + ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }, index) => { + it(`should render sound test cases #${index + 1}`, () => { + const { queryAllByTestId } = renderTestComponent( + rtcUsers, + livekitParticipantIdentities, + ); + expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); + }); + }, +); + it("should not setup audioContext gain and pan if there is no need to.", () => { - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - render( - - - , - ); + renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1); @@ -154,28 +215,8 @@ it("should setup audioContext gain and pan", () => { pan: 1, volume: 0.1, }); - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - render( - - - , - ); + + renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; expect(audioTrack.setAudioContext).toHaveBeenCalled(); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 76c206c7..24455f70 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -32,6 +32,7 @@ export interface MatrixAudioRendererProps { * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ + // TODO: Why do we have this structure? looks like we only need the valid/active participants (not the room member or id)? participants: { id: string; // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit? @@ -66,8 +67,15 @@ export function LivekitRoomAudioRenderer({ participants, muted, }: MatrixAudioRendererProps): ReactNode { - const participantSet = useMemo( - () => new Set(participants.map(({ participant }) => participant)), + // This is the list of valid identities that are allowed to play audio. + // It is derived from the list of matrix rtc members. + const validIdentities = useMemo( + () => + new Set( + participants + .filter(({ participant }) => participant) // filter out participants that are not yet joined in livekit + .map(({ participant }) => participant!.identity), + ), [participants], ); @@ -102,7 +110,7 @@ export function LivekitRoomAudioRenderer({ room: livekitRoom, }, ).filter((ref) => { - const isValid = participantSet?.has(ref.participant); + const isValid = validIdentities.has(ref.participant.identity); if (!isValid && !ref.participant.isLocal) logInvalid(ref.participant.identity); return ( @@ -115,14 +123,14 @@ export function LivekitRoomAudioRenderer({ useEffect(() => { if ( loggedInvalidIdentities.current.size && - tracks.every((t) => participantSet.has(t.participant)) + tracks.every((t) => validIdentities.has(t.participant.identity)) ) { logger.debug( `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, ); loggedInvalidIdentities.current.clear(); } - }, [tracks, participantSet, url]); + }, [tracks, validIdentities, url]); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/utils/test.ts b/src/utils/test.ts index 98a2addf..d0e08dd8 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -28,6 +28,7 @@ import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixr import { type LocalParticipant, type LocalTrackPublication, + type Participant, type RemoteParticipant, type RemoteTrackPublication, type Room as LivekitRoom, @@ -392,11 +393,9 @@ export class MockRTCSession extends TypedEventEmitter< } } -export const mockTrack = (identity: string): TrackReference => +export const mockTrack = (participant: Participant): TrackReference => ({ - participant: { - identity, - }, + participant, publication: { kind: Track.Kind.Audio, source: "mic", From 1a4b38cf930864493a4ea3d85d569de82b497cb3 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 12:51:51 -0400 Subject: [PATCH 070/144] Document ObservableScope.reconcile --- src/state/ObservableScope.ts | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 08a4b859..d6fd8e32 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -99,7 +99,19 @@ export class ObservableScope { .subscribe(callback); } - // TODO-MULTI-SFU Dear Future Robin, please document this. Love, Past Robin. + /** + * For the duration of the scope, sync some external state with the value of + * the provided Behavior by way of an async function which attempts to update + * (reconcile) the external state. The reconciliation function may return a + * clean-up callback which will be called and awaited before the next change + * in value (or the end of the scope). + * + * All calls to the function and its clean-up callbacks are serialized. If the + * value changes faster than the handlers can keep up with, intermediate + * values may be skipped. + * + * Basically, this is like React's useEffect but async and for Behaviors. + */ public reconcile( value$: Behavior, callback: (value: T) => Promise<(() => Promise) | undefined>, @@ -107,27 +119,27 @@ export class ObservableScope { let latestValue: T | typeof nothing = nothing; let reconciledValue: T | typeof nothing = nothing; let cleanUp: (() => Promise) | undefined = undefined; - let callbackPromise: Promise<(() => Promise) | undefined>; value$ .pipe( - catchError(() => EMPTY), - this.bind(), - endWith(nothing), + catchError(() => EMPTY), // Ignore errors + this.bind(), // Limit to the duration of the scope + endWith(nothing), // Clean up when the scope ends ) .subscribe((value) => { void (async (): Promise => { if (latestValue === nothing) { latestValue = value; while (latestValue !== reconciledValue) { - await cleanUp?.(); + await cleanUp?.(); // Call the previous value's clean-up handler reconciledValue = latestValue; - if (latestValue !== nothing) { - callbackPromise = callback(latestValue); - cleanUp = await callbackPromise; - } + if (latestValue !== nothing) + cleanUp = await callback(latestValue); // Sync current value } + // Reset to signal that reconciliation is done for now latestValue = nothing; } else { + // There's already an instance of the above 'while' loop running + // concurrently. Just update the latest value and let it be handled. latestValue = value; } })(); From e346c8c148bd6aad302598d57f16f02dc91cf468 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 16:39:27 -0400 Subject: [PATCH 071/144] Re-enable React strict mode --- src/main.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index f27b55a4..e6a102c6 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -11,6 +11,7 @@ Please see LICENSE in the repository root for full details. // dependency references. import "matrix-js-sdk/lib/browser-index"; +import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import "./index.css"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -59,9 +60,9 @@ if (fatalError !== null) { Initializer.initBeforeReact() .then(() => { root.render( - // - , - // , + + , + , ); }) .catch((e) => { From c96e81bfd375262be9c992cbd1885c7d878f5b8d Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 16:40:06 -0400 Subject: [PATCH 072/144] Simplify type of audio participants exposed from CallViewModel --- src/livekit/MatrixAudioRenderer.test.tsx | 21 ++++-------------- src/livekit/MatrixAudioRenderer.tsx | 27 ++++++++---------------- src/room/InCallView.tsx | 4 ++-- src/state/CallViewModel.ts | 23 ++++++++++++++++++-- 4 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index c1ee6f83..9519ccc2 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -23,12 +23,7 @@ import { useTracks } from "@livekit/components-react"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { - mockMatrixRoomMember, - mockMediaDevices, - mockRtcMembership, - mockTrack, -} from "../utils/test"; +import { mockMediaDevices, mockTrack } from "../utils/test"; export const TestAudioContextConstructor = vi.fn(() => testAudioContext); @@ -80,17 +75,11 @@ function renderTestComponent( isLocal, } as unknown as RemoteParticipant); }); - const participants = rtcMembers.map(({ userId, deviceId }) => { + const participants = rtcMembers.flatMap(({ userId, deviceId }) => { const p = liveKitParticipants.find( (p) => p.identity === `${userId}:${deviceId}`, ); - const localRtcMember = mockRtcMembership(userId, deviceId); - const member = mockMatrixRoomMember(localRtcMember); - return { - id: `${userId}:${deviceId}`, - participant: p, - member, - }; + return p === undefined ? [] : [p]; }); const livekitRoom = vi.mocked({ remoteParticipants: new Map( @@ -98,9 +87,7 @@ function renderTestComponent( ), } as unknown as Room); - tracks = participants - .filter((p) => p.participant) - .map((p) => mockTrack(p.participant!)) as TrackReference[]; + tracks = participants.map((p) => mockTrack(p)); vi.mocked(useTracks).mockReturnValue(tracks); return render( diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 24455f70..fb1400b4 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,7 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { type Room as LivekitRoom, type Participant } from "livekit-client"; +import { + type RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { @@ -14,13 +17,13 @@ import { AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; -import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; import {} from "@livekit/components-core"; + export interface MatrixAudioRendererProps { /** * The service URL of the LiveKit room. @@ -32,13 +35,7 @@ export interface MatrixAudioRendererProps { * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ - // TODO: Why do we have this structure? looks like we only need the valid/active participants (not the room member or id)? - participants: { - id: string; - // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit? - participant: Participant | undefined; - member: RoomMember; - }[]; + participants: RemoteParticipant[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -48,8 +45,7 @@ export interface MatrixAudioRendererProps { } /** - * The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app. - * It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. + * Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. * * It also takes care of the earpiece audio configuration for iOS devices. * This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio. @@ -70,12 +66,7 @@ export function LivekitRoomAudioRenderer({ // This is the list of valid identities that are allowed to play audio. // It is derived from the list of matrix rtc members. const validIdentities = useMemo( - () => - new Set( - participants - .filter(({ participant }) => participant) // filter out participants that are not yet joined in livekit - .map(({ participant }) => participant!.identity), - ), + () => new Set(participants.map((p) => p.identity)), [participants], ); @@ -92,7 +83,7 @@ export function LivekitRoomAudioRenderer({ if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, - `current members: ${participants.map((p) => p.participant?.identity)}`, + `current members: ${participants.map((p) => p.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 8474c2fd..dacb7eb1 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -286,7 +286,7 @@ export const InCallView: FC = ({ ); const allLivekitRooms = useBehavior(vm.allLivekitRooms$); - const participantsByRoom = useBehavior(vm.participantsByRoom$); + const audioParticipants = useBehavior(vm.audioParticipants$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -860,7 +860,7 @@ export const InCallView: FC = ({ ) } - {participantsByRoom.map(({ livekitRoom, url, participants }) => ( + {audioParticipants.map(({ livekitRoom, url, participants }) => ( + data.map(({ livekitRoom, url, participants }) => ({ + livekitRoom, + url, + participants: participants.flatMap(({ participant }) => + participant instanceof RemoteParticipant ? [participant] : [], + ), + })), + ), + ), + ); + /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members From 5da780ed301bc6e259ba9fb52c7b92b3aef88525 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 16:43:25 -0400 Subject: [PATCH 073/144] Remove dead MuteStates file It's been replaced by a refactored RxJS version living in src/state. --- src/room/MuteStates.test.tsx | 4 + src/room/MuteStates.ts | 179 ----------------------------------- 2 files changed, 4 insertions(+), 179 deletions(-) delete mode 100644 src/room/MuteStates.ts diff --git a/src/room/MuteStates.test.tsx b/src/room/MuteStates.test.tsx index eb08217d..d34f4d39 100644 --- a/src/room/MuteStates.test.tsx +++ b/src/room/MuteStates.test.tsx @@ -5,6 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +// TODO-MULTI-SFU: These tests need to be ported to the new MuteStates class. +/* + import { afterAll, afterEach, @@ -321,3 +324,4 @@ describe("useMuteStates in VITE_PACKAGE='embedded' (widget) mode", () => { expect(screen.getByTestId("video-enabled").textContent).toBe("true"); }); }); +*/ diff --git a/src/room/MuteStates.ts b/src/room/MuteStates.ts deleted file mode 100644 index dfc599e7..00000000 --- a/src/room/MuteStates.ts +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type Dispatch, - type SetStateAction, - useCallback, - useEffect, - useMemo, -} from "react"; -import { type IWidgetApiRequest } from "matrix-widget-api"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { useObservableEagerState } from "observable-hooks"; - -import { - type DeviceLabel, - type SelectedDevice, - type MediaDevice, -} from "../state/MediaDevices"; -import { useIsEarpiece, useMediaDevices } from "../MediaDevicesContext"; -import { useReactiveState } from "../useReactiveState"; -import { ElementWidgetActions, widget } from "../widget"; -import { Config } from "../config/Config"; -import { useUrlParams } from "../UrlParams"; - -// /** -// * If there already are this many participants in the call, we automatically mute -// * the user. -// */ -// TODO: multi-sfu dead code? -// export const MUTE_PARTICIPANT_COUNT = 8; - -interface DeviceAvailable { - enabled: boolean; - setEnabled: Dispatch>; -} - -interface DeviceUnavailable { - enabled: false; - setEnabled: null; -} - -const deviceUnavailable: DeviceUnavailable = { - enabled: false, - setEnabled: null, -}; - -type MuteState = DeviceAvailable | DeviceUnavailable; - -export interface MuteStates { - audio: MuteState; - video: MuteState; -} - -function useMuteState( - device: MediaDevice, - enabledByDefault: () => boolean, - forceUnavailable: boolean = false, -): MuteState { - const available = useObservableEagerState(device.available$); - const [enabled, setEnabled] = useReactiveState( - // Determine the default value once devices are actually connected - (prev) => prev ?? (available.size > 0 ? enabledByDefault() : undefined), - [available.size], - ); - return useMemo( - () => - available.size === 0 || forceUnavailable - ? deviceUnavailable - : { - enabled: enabled ?? false, - setEnabled: setEnabled as Dispatch>, - }, - [available.size, enabled, forceUnavailable, setEnabled], - ); -} - -export function useMuteStates(isJoined: boolean): MuteStates { - const devices = useMediaDevices(); - - const { skipLobby, defaultAudioEnabled, defaultVideoEnabled } = - useUrlParams(); - - const audio = useMuteState( - devices.audioInput, - () => - (defaultAudioEnabled ?? Config.get().media_devices.enable_audio) && - allowJoinUnmuted(skipLobby, isJoined), - ); - useEffect(() => { - // If audio is enabled, we need to request the device names again, - // because iOS will not be able to switch to the correct device after un-muting. - // This is one of the main changes that makes iOS work with bluetooth audio devices. - if (audio.enabled) { - devices.requestDeviceNames(); - } - }, [audio.enabled, devices]); - const isEarpiece = useIsEarpiece(); - const video = useMuteState( - devices.videoInput, - () => - (defaultVideoEnabled ?? Config.get().media_devices.enable_video) && - allowJoinUnmuted(skipLobby, isJoined), - isEarpiece, // Force video to be unavailable if using earpiece - ); - - useEffect(() => { - widget?.api.transport - .send(ElementWidgetActions.DeviceMute, { - audio_enabled: audio.enabled, - video_enabled: video.enabled, - }) - .catch((e) => - logger.warn("Could not send DeviceMute action to widget", e), - ); - }, [audio, video]); - - const onMuteStateChangeRequest = useCallback( - (ev: CustomEvent) => { - // First copy the current state into our new state. - const newState = { - audio_enabled: audio.enabled, - video_enabled: video.enabled, - }; - // Update new state if there are any requested changes from the widget action - // in `ev.detail.data`. - if ( - ev.detail.data.audio_enabled != null && - typeof ev.detail.data.audio_enabled === "boolean" - ) { - audio.setEnabled?.(ev.detail.data.audio_enabled); - newState.audio_enabled = ev.detail.data.audio_enabled; - } - if ( - ev.detail.data.video_enabled != null && - typeof ev.detail.data.video_enabled === "boolean" - ) { - video.setEnabled?.(ev.detail.data.video_enabled); - newState.video_enabled = ev.detail.data.video_enabled; - } - // Always reply with the new (now "current") state. - // This allows to also use this action to just get the unaltered current state - // by using a fromWidget request with: `ev.detail.data = {}` - widget!.api.transport.reply(ev.detail, newState); - }, - [audio, video], - ); - useEffect(() => { - // We setup a event listener for the widget action ElementWidgetActions.DeviceMute. - if (widget) { - // only setup the listener in widget mode - - widget.lazyActions.on( - ElementWidgetActions.DeviceMute, - onMuteStateChangeRequest, - ); - - return (): void => { - // return a call to `off` so that we always clean up our listener. - widget?.lazyActions.off( - ElementWidgetActions.DeviceMute, - onMuteStateChangeRequest, - ); - }; - } - }, [onMuteStateChangeRequest]); - - return useMemo(() => ({ audio, video }), [audio, video]); -} - -function allowJoinUnmuted(skipLobby: boolean, isJoined: boolean): boolean { - return ( - (!skipLobby && !isJoined) || import.meta.env.VITE_PACKAGE === "embedded" - ); -} From b1d143720aa322d9f263c4e3736a0dd5ebc0d468 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:08:51 -0400 Subject: [PATCH 074/144] Add comments to Async --- src/state/Async.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/state/Async.ts b/src/state/Async.ts index 79de4140..61871f78 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -7,8 +7,12 @@ Please see LICENSE in the repository root for full details. import { catchError, from, map, type Observable, of, startWith } from "rxjs"; -// TODO where are all the comments? ::cry:: -// There used to be an unitialized state!, a state might not start in loading +/** + * Data that may need to be loaded asynchronously. + * + * This type is for when you need to represent the current state of an operation + * involving Promises as **immutable data**. See the async$ function below. + */ export type Async = | { state: "loading" } | { state: "error"; value: Error } @@ -23,6 +27,11 @@ export function ready(value: A): Async { return { state: "ready", value }; } +/** + * Turn a Promise into an Observable async value. The Observable will have the + * value "loading" while the Promise is pending, "ready" when the Promise + * resolves, and "error" when the Promise rejects. + */ export function async$(promise: Promise): Observable> { return from(promise).pipe( map(ready), @@ -33,6 +42,9 @@ export function async$(promise: Promise): Observable> { ); } +/** + * If the async value is ready, apply the given function to the inner value. + */ export function mapAsync( async: Async, project: (value: A) => B, From e88474452fddb49432d47dd13cbc0f6e3c8eab7e Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:33:17 -0400 Subject: [PATCH 075/144] Correct / document some missing bits in tests --- src/room/GroupCallView.test.tsx | 2 ++ src/room/InCallView.test.tsx | 1 + src/room/InCallView.tsx | 2 +- src/room/VideoPreview.test.tsx | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 8c4a276a..ea14f5cf 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -150,6 +150,7 @@ function createGroupCallView( const muteState = { audio: { enabled: false }, video: { enabled: false }, + // TODO-MULTI-SFU: This cast isn't valid, it's likely the cause of some current test failures } as unknown as MuteStates; const { getByText } = render( @@ -166,6 +167,7 @@ function createGroupCallView( rtcSession={rtcSession as unknown as MatrixRTCSession} muteStates={muteState} widget={widget} + // TODO-MULTI-SFU: Make joined and setJoined work joined={true} setJoined={function (value: boolean): void {}} /> diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index d2694120..131259da 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -136,6 +136,7 @@ function createInCallView(): RenderResult & { const muteState = { audio: { enabled: false }, video: { enabled: false }, + // TODO-MULTI-SFU: This cast isn't valid, it's likely the cause of some current test failures } as unknown as MuteStates; const livekitRoom = mockLivekitRoom( { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index dacb7eb1..658f9fbe 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -205,7 +205,7 @@ export const InCallView: FC = ({ useReactionsSender(); useWakeLock(); - // TODO multi-sfu This is unused now?? + // TODO-MULTI-SFU This is unused now?? // const connectionState = useObservableEagerState(vm.livekitConnectionState$); // annoyingly we don't get the disconnection reason this way, diff --git a/src/room/VideoPreview.test.tsx b/src/room/VideoPreview.test.tsx index 17a05e34..dba65727 100644 --- a/src/room/VideoPreview.test.tsx +++ b/src/room/VideoPreview.test.tsx @@ -41,7 +41,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, @@ -53,7 +53,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, From 8778be83510a93d9c4e89dab93b25467020737f0 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:34:04 -0400 Subject: [PATCH 076/144] Fix doc comment typo --- src/state/PublishConnection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index c35c71e4..9a219483 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -36,7 +36,7 @@ import { Connection, type ConnectionOpts } from "./Connection.ts"; import { type ObservableScope } from "./ObservableScope.ts"; /** - * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. + * A connection to the local LiveKit room, the one the user is publishing to. * This connection will publish the local user's audio and video tracks. */ export class PublishConnection extends Connection { From 3691e7120d572082af5560aa5539aa02a5eb6906 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:35:53 -0400 Subject: [PATCH 077/144] Restore a hidden 'null' state for the local transport/connection --- src/state/CallViewModel.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4d83d997..815cbf17 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -562,16 +562,22 @@ export class CallViewModel extends ViewModel { /** * The transport over which we should be actively publishing our media. + * null when not joined. */ - private readonly localTransport$: Behavior> = + private readonly localTransport$: Behavior | null> = this.scope.behavior( this.transports$.pipe( map((transports) => transports?.local ?? null), - distinctUntilChanged(deepCompare), + distinctUntilChanged | null>(deepCompare), ), ); - private readonly localConnection$: Behavior> = + /** + * The local connection over which we will publish our media. It could + * possibly also have some remote users' media available on it. + * null when not joined. + */ + private readonly localConnection$: Behavior | null> = this.scope.behavior( this.localTransport$.pipe( map( @@ -807,15 +813,14 @@ export class CallViewModel extends ViewModel { // TODO: Move this logic into Connection/PublishConnection if possible this.localConnection$ .pipe( - switchMap((values) => { - if (values?.state !== "ready") return []; - const localConnection = values.value; + switchMap((localConnection) => { + if (localConnection?.state !== "ready") return []; const memberError = (): never => { throw new Error("No room member for call membership"); }; const localParticipant = { id: "local", - participant: localConnection.livekitRoom.localParticipant, + participant: localConnection.value.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), }; @@ -823,7 +828,7 @@ export class CallViewModel extends ViewModel { return this.remoteConnections$.pipe( switchMap((remoteConnections) => combineLatest( - [localConnection, ...remoteConnections].map((c) => + [localConnection.value, ...remoteConnections].map((c) => c.publishingParticipants$.pipe( map((ps) => { const participants: { @@ -842,7 +847,7 @@ export class CallViewModel extends ViewModel { this.matrixRoom, )?.member ?? memberError(), })); - if (c === localConnection) + if (c === localConnection.value) participants.push(localParticipant); return { @@ -1974,7 +1979,10 @@ export class CallViewModel extends ViewModel { ); c.stop().catch((err) => { // TODO: better error handling - logger.error("MuteState: handler error", err); + logger.error( + `Fail to stop connection to ${c.localTransport.livekit_service_url}`, + err, + ); }); } for (const c of start) { From dee06a4b701ba8ca3b46ab70f575d93a85b3fea9 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:37:54 -0400 Subject: [PATCH 078/144] Remove unused useIsEarpiece hook --- src/MediaDevicesContext.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/MediaDevicesContext.ts b/src/MediaDevicesContext.ts index 3cf54c2a..801219b0 100644 --- a/src/MediaDevicesContext.ts +++ b/src/MediaDevicesContext.ts @@ -23,14 +23,6 @@ export function useMediaDevices(): MediaDevices { return mediaDevices; } -export const useIsEarpiece = (): boolean => { - const devices = useMediaDevices(); - const audioOutput = useObservableEagerState(devices.audioOutput.selected$); - const available = useObservableEagerState(devices.audioOutput.available$); - if (!audioOutput?.id) return false; - return available.get(audioOutput.id)?.type === "earpiece"; -}; - /** * A convenience hook to get the audio node configuration for the earpiece. * It will check the `useAsEarpiece` of the `audioOutput` device and return From dcc3ab641f59ae986edca267d9b7787ed9c546fa Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:40:03 -0400 Subject: [PATCH 079/144] Remove MockedObject from mockMediaDevices type signature --- src/utils/test.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/utils/test.ts b/src/utils/test.ts index d0e08dd8..6e0e95c9 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, type MockedObject, vi, vitest } from "vitest"; +import { expect, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -418,15 +418,13 @@ export const deviceStub = { select(): void {}, }; -export function mockMediaDevices( - data: Partial, -): MockedObject { - return vi.mocked({ +export function mockMediaDevices(data: Partial): MediaDevices { + return { audioInput: deviceStub, audioOutput: deviceStub, videoInput: deviceStub, ...data, - } as MediaDevices); + } as MediaDevices; } export function mockMuteStates( From 00daf834b655140d402515beb52505946e79b7d0 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:53:11 -0400 Subject: [PATCH 080/144] Remove local participant case (now enforced by types) from audio tests --- src/livekit/MatrixAudioRenderer.test.tsx | 44 +++++++----------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 9519ccc2..b78b274d 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -65,27 +65,25 @@ let tracks: TrackReference[] = []; function renderTestComponent( rtcMembers: { userId: string; deviceId: string }[], - livekitParticipantIdentities: ({ id: string; isLocal?: boolean } | string)[], + livekitParticipantIdentities: string[], ): RenderResult { - const liveKitParticipants = livekitParticipantIdentities.map((p) => { - const identity = typeof p === "string" ? p : p.id; - const isLocal = typeof p === "string" ? false : (p.isLocal ?? false); - return vi.mocked({ - identity, - isLocal, - } as unknown as RemoteParticipant); - }); + const liveKitParticipants = livekitParticipantIdentities.map( + (identity) => + ({ + identity, + }) as unknown as RemoteParticipant, + ); const participants = rtcMembers.flatMap(({ userId, deviceId }) => { const p = liveKitParticipants.find( (p) => p.identity === `${userId}:${deviceId}`, ); return p === undefined ? [] : [p]; }); - const livekitRoom = vi.mocked({ + const livekitRoom = { remoteParticipants: new Map( liveKitParticipants.map((p) => [p.identity, p]), ), - } as unknown as Room); + } as unknown as Room; tracks = participants.map((p) => mockTrack(p)); @@ -121,7 +119,7 @@ it("should not render without member", () => { const TEST_CASES: { rtcUsers: { userId: string; deviceId: string }[]; - livekitParticipantIdentities: (string | { id: string; isLocal?: boolean })[]; + livekitParticipantIdentities: string[]; expectedAudioTracks: number; }[] = [ { @@ -130,27 +128,9 @@ const TEST_CASES: { { userId: "@alice", deviceId: "DEV1" }, { userId: "@bob", deviceId: "DEV0" }, ], - livekitParticipantIdentities: [ - { id: "@alice:DEV0" }, - "@bob:DEV0", - "@alice:DEV1", - ], + livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@alice:DEV1"], expectedAudioTracks: 3, }, - // Alice DEV0 is local participant, should not render - { - rtcUsers: [ - { userId: "@alice", deviceId: "DEV0" }, - { userId: "@alice", deviceId: "DEV1" }, - { userId: "@bob", deviceId: "DEV0" }, - ], - livekitParticipantIdentities: [ - { id: "@alice:DEV0", isLocal: true }, - "@bob:DEV0", - "@alice:DEV1", - ], - expectedAudioTracks: 2, - }, // Charlie is a rtc member but not in livekit { rtcUsers: [ @@ -158,7 +138,7 @@ const TEST_CASES: { { userId: "@bob", deviceId: "DEV0" }, { userId: "@charlie", deviceId: "DEV0" }, ], - livekitParticipantIdentities: ["@alice:DEV0", { id: "@bob:DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0"], expectedAudioTracks: 2, }, // Charlie is in livekit but not rtc member From 2d7e4247f31f0c7226e4aa6fe979d607e287c55a Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:51:30 -0400 Subject: [PATCH 081/144] Note a potential resource leak --- src/livekit/TrackProcessorContext.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index 4a5ace46..bb13d5dc 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -67,6 +67,7 @@ export const trackProcessorSync = ( videoTrack$: Behavior, processor$: Behavior, ): void => { + // TODO-MULTI-SFU: Bind to an ObservableScope to avoid leaking resources. combineLatest([videoTrack$, processor$]).subscribe( ([videoTrack, processorState]) => { if (!processorState) return; From 5be3b9150959b3fa15396014ca7493f258101d3f Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:10:26 -0400 Subject: [PATCH 082/144] Fix focus connection state typo, simplify its initialization --- src/state/CallViewModel.ts | 2 +- src/state/Connection.test.ts | 28 ++++++---------------------- src/state/Connection.ts | 25 ++++++++++--------------- 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 815cbf17..4cb97519 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -607,7 +607,7 @@ export class CallViewModel extends ViewModel { switchMap((c) => c?.state === "ready" ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? - c.value.focusedConnectionState$.pipe( + c.value.focusConnectionState$.pipe( map((s) => { if (s.state === "ConnectedToLkRoom") return s.connectionState; return ConnectionState.Disconnected; diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 69942270..ecafb5ee 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -159,7 +159,7 @@ describe("Start connection states", () => { }; const connection = new RemoteConnection(opts, undefined); - expect(connection.focusedConnectionState$.getValue().state).toEqual( + expect(connection.focusConnectionState$.getValue().state).toEqual( "Initialized", ); }); @@ -179,7 +179,7 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); @@ -231,7 +231,7 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); @@ -287,7 +287,7 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); @@ -343,7 +343,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); const capturedState: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -368,7 +368,7 @@ describe("Start connection states", () => { await connection.start(); let capturedState: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -417,12 +417,6 @@ describe("Start connection states", () => { vi.useFakeTimers(); const connection = setupRemoteConnection(); - - let capturedState: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { - capturedState.push(value); - }); - await connection.start(); const stopSpy = vi.spyOn(connection, "stop"); @@ -430,16 +424,6 @@ describe("Start connection states", () => { expect(stopSpy).toHaveBeenCalled(); expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); - - /// Ensures that focusedConnectionState$ is bound to the scope. - capturedState = []; - // the subscription should be closed, and no new state should be received - // @ts-expect-error: Accessing private field for testing purposes - connection._focusedConnectionState$.next({ state: "Initialized" }); - // @ts-expect-error: Accessing private field for testing purposes - connection._focusedConnectionState$.next({ state: "ConnectingToLkRoom" }); - - expect(capturedState.length).toEqual(0); }); }); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index e5e108b7..ac381d56 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -66,13 +66,14 @@ export type FocusConnectionState = */ export class Connection { // Private Behavior - private readonly _focusedConnectionState$ = + private readonly _focusConnectionState$ = new BehaviorSubject({ state: "Initialized" }); /** * The current state of the connection to the focus server. */ - public readonly focusedConnectionState$: Behavior; + public readonly focusConnectionState$: Behavior = + this._focusConnectionState$; /** * Whether the connection has been stopped. @@ -91,7 +92,7 @@ export class Connection { public async start(): Promise { this.stopped = false; try { - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "FetchingConfig", focus: this.localTransport, }); @@ -100,7 +101,7 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "ConnectingToLkRoom", focus: this.localTransport, }); @@ -108,13 +109,13 @@ export class Connection { // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "ConnectedToLkRoom", focus: this.localTransport, connectionState: this.livekitRoom.state, }); } catch (error) { - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "FailedToStart", error: error instanceof Error ? error : new Error(`${error}`), focus: this.localTransport, @@ -139,7 +140,7 @@ export class Connection { public async stop(): Promise { if (this.stopped) return; await this.livekitRoom.disconnect(); - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "Stopped", focus: this.localTransport, }); @@ -172,15 +173,9 @@ export class Connection { ) { const { transport, client, scope, remoteTransports$ } = opts; - this.livekitRoom = livekitRoom; this.localTransport = transport; this.client = client; - this.focusedConnectionState$ = scope.behavior( - this._focusedConnectionState$, - { state: "Initialized" }, - ); - const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), [], @@ -212,10 +207,10 @@ export class Connection { scope .behavior(connectionStateObserver(this.livekitRoom)) .subscribe((connectionState) => { - const current = this._focusedConnectionState$.value; + const current = this._focusConnectionState$.value; // Only update the state if we are already connected to the LiveKit room. if (current.state === "ConnectedToLkRoom") { - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "ConnectedToLkRoom", connectionState, focus: current.focus, From 64c2e5911c37f3ff73d8966756e721768df0567e Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:17:42 -0400 Subject: [PATCH 083/144] Update outdated comment --- src/state/Connection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/Connection.ts b/src/state/Connection.ts index ac381d56..b7864677 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -149,7 +149,7 @@ export class Connection { /** * An observable of the participants that are publishing on this connection. - * This is derived from `participantsIncludingSubscribers$` and `membershipsFocusMap$`. + * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ public readonly publishingParticipants$; From 2c576a7477d509a293e9bf59b48359934816cf26 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:58:03 -0400 Subject: [PATCH 084/144] Clean up subscriptions in Connection tests --- src/state/Connection.test.ts | 44 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index ecafb5ee..14c42206 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -12,6 +12,7 @@ import { it, type Mock, type MockedObject, + onTestFinished, vi, } from "vitest"; import { BehaviorSubject, of } from "rxjs"; @@ -179,9 +180,10 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { + const s = connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const deferred = Promise.withResolvers(); @@ -231,9 +233,10 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { + const s = connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call @@ -287,9 +290,10 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { + const s = connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call @@ -342,21 +346,22 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedState: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { - capturedState.push(value); + const capturedStates: FocusConnectionState[] = []; + const s = connection.focusConnectionState$.subscribe((value) => { + capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); await connection.start(); await vi.runAllTimersAsync(); - const initialState = capturedState.shift(); + const initialState = capturedStates.shift(); expect(initialState?.state).toEqual("Initialized"); - const fetchingState = capturedState.shift(); + const fetchingState = capturedStates.shift(); expect(fetchingState?.state).toEqual("FetchingConfig"); - const connectingState = capturedState.shift(); + const connectingState = capturedStates.shift(); expect(connectingState?.state).toEqual("ConnectingToLkRoom"); - const connectedState = capturedState.shift(); + const connectedState = capturedStates.shift(); expect(connectedState?.state).toEqual("ConnectedToLkRoom"); }); @@ -367,10 +372,11 @@ describe("Start connection states", () => { await connection.start(); - let capturedState: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { - capturedState.push(value); + let capturedStates: FocusConnectionState[] = []; + const s = connection.focusConnectionState$.subscribe((value) => { + capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const states = [ ConnectionState.Disconnected, @@ -386,7 +392,7 @@ describe("Start connection states", () => { } for (const state of states) { - const s = capturedState.shift(); + const s = capturedStates.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); const connectedState = s as FocusConnectionState & { state: "ConnectedToLkRoom"; @@ -404,12 +410,12 @@ describe("Start connection states", () => { // If the state is not ConnectedToLkRoom, no events should be relayed anymore await connection.stop(); - capturedState = []; + capturedStates = []; for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); } - expect(capturedState.length).toEqual(0); + expect(capturedStates.length).toEqual(0); }); it("shutting down the scope should stop the connection", async () => { @@ -452,7 +458,7 @@ describe("Publishing participants observations", () => { participant: RemoteParticipant; membership: CallMembership; }[][] = []; - connection.publishingParticipants$.subscribe((publishers) => { + const s = connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); if ( publishers.some( @@ -469,6 +475,7 @@ describe("Publishing participants observations", () => { danIsAPublisher.resolve(); } }); + onTestFinished(() => s.unsubscribe()); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. @@ -578,9 +585,10 @@ describe("Publishing participants observations", () => { participant: RemoteParticipant; membership: CallMembership; }[][] = []; - connection.publishingParticipants$.subscribe((publishers) => { + const s = connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); }); + onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), From 85ffe68d98b887b4e6f60601977e84f223e04d1a Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 19:20:21 -0400 Subject: [PATCH 085/144] Remove outdated comment --- src/state/MuteStates.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 8a025882..50be5e05 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -138,7 +138,6 @@ class MuteState { ) {} } -// TODO there is another MuteStates in src/room/MuteStates.tsx ?? why export class MuteStates { public readonly audio = new MuteState( this.scope, From 4c6b960da34e98df5f43cf5e028fc98755fe31c6 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 9 Oct 2025 11:00:45 +0200 Subject: [PATCH 086/144] fix: use correct TestEachFunction --- src/livekit/MatrixAudioRenderer.test.tsx | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index b78b274d..83b3f73a 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -152,15 +152,14 @@ const TEST_CASES: { }, ]; -TEST_CASES.forEach( - ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }, index) => { - it(`should render sound test cases #${index + 1}`, () => { - const { queryAllByTestId } = renderTestComponent( - rtcUsers, - livekitParticipantIdentities, - ); - expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); - }); +it.each(TEST_CASES)( + `should render sound test cases %s`, + ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }) => { + const { queryAllByTestId } = renderTestComponent( + rtcUsers, + livekitParticipantIdentities, + ); + expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); }, ); From 39f8cb95ec03f2d75c2979fa4fd7812bdf367fee Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 9 Oct 2025 15:29:47 +0200 Subject: [PATCH 087/144] make compatible with related_event (async CallMembership) branch Signed-off-by: Timo K --- package.json | 2 +- src/home/useGroupCallRooms.ts | 35 +++++++++++++++++++---------------- yarn.lock | 10 +++++----- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 91583023..571fe9e7 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/home/useGroupCallRooms.ts b/src/home/useGroupCallRooms.ts index e89c3f14..149af4b0 100644 --- a/src/home/useGroupCallRooms.ts +++ b/src/home/useGroupCallRooms.ts @@ -20,6 +20,7 @@ import { MatrixRTCSessionManagerEvents, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; import { getKeyForRoom } from "../e2ee/sharedKeyManagement"; @@ -139,22 +140,24 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { .filter(roomHasCallMembershipEvents) .filter(roomIsJoinable); const sortedRooms = sortRooms(client, rooms); - const items = sortedRooms.map((room) => { - const session = client.matrixRTC.getRoomSession(room); - return { - roomAlias: room.getCanonicalAlias() ?? undefined, - roomName: room.name, - avatarUrl: room.getMxcAvatarUrl()!, - room, - session, - participants: session.memberships - .filter((m) => m.sender) - .map((m) => room.getMember(m.sender!)) - .filter((m) => m) as RoomMember[], - }; - }); - - setRooms(items); + Promise.all( + sortedRooms.map(async (room) => { + const session = await client.matrixRTC.getRoomSession(room); + return { + roomAlias: room.getCanonicalAlias() ?? undefined, + roomName: room.name, + avatarUrl: room.getMxcAvatarUrl()!, + room, + session, + participants: session.memberships + .filter((m) => m.sender) + .map((m) => room.getMember(m.sender!)) + .filter((m) => m) as RoomMember[], + }; + }), + ) + .then((items) => setRooms(items)) + .catch(logger.error); } updateRooms(); diff --git a/yarn.lock b/yarn.lock index 197cee3e..701a7f67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7537,7 +7537,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10297,9 +10297,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": - version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=a343e8c92a5a37f419eb1b762db3a123e41ef66d" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts": + version: 38.4.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4608506288c6beaa252982d224e996e23e51f681" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10315,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/4893878f2fe07b06334bab4674a01569037d0f3e737fef3f0bb97a98b01d71fc304627921673f128821a17d824de9b63cc06456db15f9d45eb10bba1ceacd5c5 + checksum: 10c0/2e896d6a92cb3bbb47c120a39dd1a0030b4bf02289cb914f6c848b564208f421ada605e8efb68f6d9d55a0d2e3f86698b6076cb029e9bab2bac0f70f7250dd17 languageName: node linkType: hard From 7cbb1ec1e8069453d987c215ec7a9cd9a2045064 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 9 Oct 2025 15:33:25 +0200 Subject: [PATCH 088/144] Simplify AudioRenderer and add more tests --- src/livekit/MatrixAudioRenderer.test.tsx | 110 ++++++++++++++++++++--- src/livekit/MatrixAudioRenderer.tsx | 83 ++++++----------- src/room/InCallView.tsx | 2 +- src/utils/test.ts | 12 ++- 4 files changed, 133 insertions(+), 74 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 83b3f73a..049add97 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -14,8 +14,8 @@ import { import { type Participant, type RemoteAudioTrack, - type RemoteParticipant, type Room, + Track, } from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; @@ -23,7 +23,11 @@ import { useTracks } from "@livekit/components-react"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { mockMediaDevices, mockTrack } from "../utils/test"; +import { + mockMediaDevices, + mockRemoteParticipant, + mockTrack, +} from "../utils/test"; export const TestAudioContextConstructor = vi.fn(() => testAudioContext); @@ -61,17 +65,20 @@ let tracks: TrackReference[] = []; * * @param rtcMembers - Array of active rtc members with userId and deviceId. * @param livekitParticipantIdentities - Array of livekit participant (that are publishing). + * @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created. * */ function renderTestComponent( rtcMembers: { userId: string; deviceId: string }[], livekitParticipantIdentities: string[], + explicitTracks?: { + participantId: string; + kind: Track.Kind; + source: Track.Source; + }[], ): RenderResult { - const liveKitParticipants = livekitParticipantIdentities.map( - (identity) => - ({ - identity, - }) as unknown as RemoteParticipant, + const liveKitParticipants = livekitParticipantIdentities.map((identity) => + mockRemoteParticipant({ identity }), ); const participants = rtcMembers.flatMap(({ userId, deviceId }) => { const p = liveKitParticipants.find( @@ -85,13 +92,22 @@ function renderTestComponent( ), } as unknown as Room; - tracks = participants.map((p) => mockTrack(p)); + if (explicitTracks?.length ?? 0 > 0) { + tracks = explicitTracks!.map(({ participantId, source, kind }) => { + const participant = + liveKitParticipants.find((p) => p.identity === participantId) ?? + mockRemoteParticipant({ identity: participantId }); + return mockTrack(participant, kind, source); + }); + } else { + tracks = participants.map((p) => mockTrack(p)); + } vi.mocked(useTracks).mockReturnValue(tracks); return render( p.identity)} livekitRoom={livekitRoom} url={""} /> @@ -118,11 +134,18 @@ it("should not render without member", () => { }); const TEST_CASES: { + name: string; rtcUsers: { userId: string; deviceId: string }[]; livekitParticipantIdentities: string[]; + explicitTracks?: { + participantId: string; + kind: Track.Kind; + source: Track.Source; + }[]; expectedAudioTracks: number; }[] = [ { + name: "single user single device", rtcUsers: [ { userId: "@alice", deviceId: "DEV0" }, { userId: "@alice", deviceId: "DEV1" }, @@ -133,6 +156,7 @@ const TEST_CASES: { }, // Charlie is a rtc member but not in livekit { + name: "Charlie is rtc member but not in livekit", rtcUsers: [ { userId: "@alice", deviceId: "DEV0" }, { userId: "@bob", deviceId: "DEV0" }, @@ -143,6 +167,7 @@ const TEST_CASES: { }, // Charlie is in livekit but not rtc member { + name: "Charlie is in livekit but not rtc member", rtcUsers: [ { userId: "@alice", deviceId: "DEV0" }, { userId: "@bob", deviceId: "DEV0" }, @@ -150,14 +175,77 @@ const TEST_CASES: { livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"], expectedAudioTracks: 2, }, + { + name: "no audio track, only video track", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV0", + kind: Track.Kind.Video, + source: Track.Source.Camera, + }, + ], + expectedAudioTracks: 0, + }, + { + name: "Audio track from unknown source", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV0", + kind: Track.Kind.Audio, + source: Track.Source.Unknown, + }, + ], + expectedAudioTracks: 1, + }, + { + name: "Audio track from other device", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV1", + kind: Track.Kind.Audio, + source: Track.Source.Microphone, + }, + ], + expectedAudioTracks: 0, + }, + { + name: "two audio tracks, microphone and screenshare", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV0", + kind: Track.Kind.Audio, + source: Track.Source.Microphone, + }, + { + participantId: "@alice:DEV0", + kind: Track.Kind.Audio, + source: Track.Source.ScreenShareAudio, + }, + ], + expectedAudioTracks: 2, + }, ]; it.each(TEST_CASES)( - `should render sound test cases %s`, - ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }) => { + `should render sound test cases $name`, + ({ + rtcUsers, + livekitParticipantIdentities, + explicitTracks, + expectedAudioTracks, + }) => { const { queryAllByTestId } = renderTestComponent( rtcUsers, livekitParticipantIdentities, + explicitTracks, ); expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); }, diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index fb1400b4..5b1149e9 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,23 +6,20 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; +import { type Room as LivekitRoom } from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { useEffect, useMemo, useState, type ReactNode } from "react"; import { useTracks, AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; -import {} from "@livekit/components-core"; export interface MatrixAudioRendererProps { /** @@ -31,11 +28,11 @@ export interface MatrixAudioRendererProps { url: string; livekitRoom: LivekitRoom; /** - * The list of participants to render audio for. + * The list of participant identities to render audio for. * This list needs to be composed based on the matrixRTC members so that we do not play audio from users - * that are not expected to be in the rtc session. + * that are not expected to be in the rtc session (local user is excluded). */ - participants: RemoteParticipant[]; + validIdentities: ParticipantId[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -44,6 +41,7 @@ export interface MatrixAudioRendererProps { muted?: boolean; } +const prefixedLogger = logger.getChild("[MatrixAudioRenderer]"); /** * Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. * @@ -60,35 +58,9 @@ export interface MatrixAudioRendererProps { export function LivekitRoomAudioRenderer({ url, livekitRoom, - participants, + validIdentities, muted, }: MatrixAudioRendererProps): ReactNode { - // This is the list of valid identities that are allowed to play audio. - // It is derived from the list of matrix rtc members. - const validIdentities = useMemo( - () => new Set(participants.map((p) => p.identity)), - [participants], - ); - - const loggedInvalidIdentities = useRef(new Set()); - - /** - * Log an invalid livekit track identity. - * A invalid identity is one that does not match any of the matrix rtc members. - * - * @param identity The identity of the track that is invalid - * @param validIdentities The list of valid identities - */ - const logInvalid = (identity: string): void => { - if (loggedInvalidIdentities.current.has(identity)) return; - logger.warn( - `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, - `current members: ${participants.map((p) => p.identity)}`, - `track will not get rendered`, - ); - loggedInvalidIdentities.current.add(identity); - }; - const tracks = useTracks( [ Track.Source.Microphone, @@ -100,28 +72,23 @@ export function LivekitRoomAudioRenderer({ onlySubscribed: true, room: livekitRoom, }, - ).filter((ref) => { - const isValid = validIdentities.has(ref.participant.identity); - if (!isValid && !ref.participant.isLocal) - logInvalid(ref.participant.identity); - return ( - !ref.participant.isLocal && - ref.publication.kind === Track.Kind.Audio && - isValid - ); - }); - - useEffect(() => { - if ( - loggedInvalidIdentities.current.size && - tracks.every((t) => validIdentities.has(t.participant.identity)) - ) { - logger.debug( - `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, - ); - loggedInvalidIdentities.current.clear(); - } - }, [tracks, validIdentities, url]); + ) + // Only keep audio tracks + .filter((ref) => ref.publication.kind === Track.Kind.Audio) + // Only keep tracks from participants that are in the validIdentities list + .filter((ref) => { + const isValid = validIdentities.includes(ref.participant.identity); + if (!isValid) { + // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. + prefixedLogger.warn( + `Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`, + `current members: ${validIdentities.join()}`, + `track will not get rendered`, + ); + return false; + } + return true; + }); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 658f9fbe..fd631bae 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -865,7 +865,7 @@ export const InCallView: FC = ({ key={url} url={url} livekitRoom={livekitRoom} - participants={participants} + validIdentities={participants.map((p) => p.identity)} muted={muteAllAudio} /> ))} diff --git a/src/utils/test.ts b/src/utils/test.ts index 6e0e95c9..508559c2 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -393,13 +393,17 @@ export class MockRTCSession extends TypedEventEmitter< } } -export const mockTrack = (participant: Participant): TrackReference => +export const mockTrack = ( + participant: Participant, + kind?: Track.Kind, + source?: Track.Source, +): TrackReference => ({ participant, publication: { - kind: Track.Kind.Audio, - source: "mic", - trackSid: "123", + kind: kind ?? Track.Kind.Audio, + source: source ?? Track.Source.Microphone, + trackSid: `123##${participant.identity}`, track: { attach: vi.fn(), detach: vi.fn(), From a500915c436415ed91b9130820da492bab3d9876 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 9 Oct 2025 19:24:44 +0200 Subject: [PATCH 089/144] test: Fix mute test, behavior change from setMuted to setAudioEnabled useCallViewKeyboardShortcuts() changed a param from `setMicrophoneMuted` to `setAudioEnabled`, the boolean arg of the callback is inverse tht it used to be --- src/useCallViewKeyboardShortcuts.test.tsx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/useCallViewKeyboardShortcuts.test.tsx b/src/useCallViewKeyboardShortcuts.test.tsx index 86e1b03f..e22380d1 100644 --- a/src/useCallViewKeyboardShortcuts.test.tsx +++ b/src/useCallViewKeyboardShortcuts.test.tsx @@ -23,14 +23,14 @@ import { // The TestComponent just wraps a button around that hook. interface TestComponentProps { - setMicrophoneMuted?: (muted: boolean) => void; + setAudioEnabled?: (enabled: boolean) => void; onButtonClick?: () => void; sendReaction?: () => void; toggleHandRaised?: () => void; } const TestComponent: FC = ({ - setMicrophoneMuted = (): void => {}, + setAudioEnabled = (): void => {}, onButtonClick = (): void => {}, sendReaction = (reaction: ReactionOption): void => {}, toggleHandRaised = (): void => {}, @@ -40,7 +40,7 @@ const TestComponent: FC = ({ ref, () => {}, () => {}, - setMicrophoneMuted, + setAudioEnabled, sendReaction, toggleHandRaised, ); @@ -57,12 +57,13 @@ test("spacebar unmutes", async () => { render( (muted = false)} - setMicrophoneMuted={(m) => { - muted = m; + setAudioEnabled={(m) => { + muted = !m; }} />, ); + expect(muted).toBe(true); await user.keyboard("[Space>]"); expect(muted).toBe(false); await user.keyboard("[/Space]"); @@ -73,15 +74,15 @@ test("spacebar unmutes", async () => { test("spacebar prioritizes pressing a button", async () => { const user = userEvent.setup(); - const setMuted = vi.fn(); + const setAudioEnabled = vi.fn(); const onClick = vi.fn(); render( - , + , ); await user.tab(); // Focus the button await user.keyboard("[Space]"); - expect(setMuted).not.toBeCalled(); + expect(setAudioEnabled).not.toBeCalled(); expect(onClick).toBeCalled(); }); @@ -129,7 +130,7 @@ test("unmuting happens in place of the default action", async () => { tabIndex={0} onKeyDown={(e) => defaultPrevented(e.isDefaultPrevented())} > - {}} /> + {}} /> , ); From 6710f4c72ae54b5ba570b057e0314b618a32a688 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 10 Oct 2025 11:09:41 +0200 Subject: [PATCH 090/144] Test: Fix mocking to fix failing tests --- src/button/ReactionToggleButton.test.tsx | 3 +-- src/reactions/ReactionsReader.test.tsx | 21 +++++++-------------- src/room/GroupCallView.test.tsx | 6 +++--- src/room/InCallView.test.tsx | 5 ++--- src/state/CallViewModel.test.ts | 3 +-- src/state/CallViewModel.ts | 14 ++++++++++---- src/utils/test-viewmodel.ts | 11 ++++------- src/utils/test.ts | 24 ++++++++++++++++++++++-- 8 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/button/ReactionToggleButton.test.tsx b/src/button/ReactionToggleButton.test.tsx index 269eabed..b1af7ec8 100644 --- a/src/button/ReactionToggleButton.test.tsx +++ b/src/button/ReactionToggleButton.test.tsx @@ -10,7 +10,6 @@ import { expect, test } from "vitest"; import { TooltipProvider } from "@vector-im/compound-web"; import { userEvent } from "@testing-library/user-event"; import { type ReactNode } from "react"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { ReactionToggleButton } from "./ReactionToggleButton"; import { ElementCallReactionEventType } from "../reactions"; @@ -33,7 +32,7 @@ function TestComponent({ diff --git a/src/reactions/ReactionsReader.test.tsx b/src/reactions/ReactionsReader.test.tsx index b8acf5c7..01815c82 100644 --- a/src/reactions/ReactionsReader.test.tsx +++ b/src/reactions/ReactionsReader.test.tsx @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { renderHook } from "@testing-library/react"; import { afterEach, test, vitest } from "vitest"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { RoomEvent as MatrixRoomEvent, MatrixEvent, @@ -38,7 +37,7 @@ test("handles a hand raised reaction", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("ab", { a: () => {}, @@ -86,7 +85,7 @@ test("handles a redaction", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("abc", { a: () => {}, @@ -149,7 +148,7 @@ test("handles waiting for event decryption", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("abc", { a: () => {}, @@ -218,7 +217,7 @@ test("hands rejecting events without a proper membership", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("ab", { a: () => {}, @@ -262,9 +261,7 @@ test("handles a reaction", () => { withTestScheduler(({ schedule, time, expectObservable }) => { renderHook(() => { - const { reactions$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, - ); + const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession()); schedule(`abc`, { a: () => {}, b: () => { @@ -320,9 +317,7 @@ test("ignores bad reaction events", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { - const { reactions$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, - ); + const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession()); schedule("ab", { a: () => {}, b: () => { @@ -444,9 +439,7 @@ test("that reactions cannot be spammed", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { - const { reactions$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, - ); + const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession()); schedule("abcd", { a: () => {}, b: () => { diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index ea14f5cf..37f5c850 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -117,7 +117,7 @@ function createGroupCallView( widget: WidgetHelpers | null, joined = true, ): { - rtcSession: MockRTCSession; + rtcSession: MatrixRTCSession; getByText: ReturnType["getByText"]; } { const client = { @@ -164,7 +164,7 @@ function createGroupCallView( preload={false} skipLobby={false} header={HeaderStyle.Standard} - rtcSession={rtcSession as unknown as MatrixRTCSession} + rtcSession={rtcSession.asMockedSession()} muteStates={muteState} widget={widget} // TODO-MULTI-SFU: Make joined and setJoined work @@ -178,7 +178,7 @@ function createGroupCallView( ); return { getByText, - rtcSession, + rtcSession: rtcSession.asMockedSession(), }; } diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 131259da..6b897c0d 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -15,7 +15,6 @@ import { } from "vitest"; import { act, render, type RenderResult } from "@testing-library/react"; import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; import { type LocalParticipant } from "livekit-client"; import { of } from "rxjs"; @@ -154,14 +153,14 @@ function createInCallView(): RenderResult & { >({}); const vm = new CallViewModel( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), room, mediaDevices, muteStates, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4cb97519..c3cf7ff3 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -541,7 +541,9 @@ export class CallViewModel extends ViewModel { const oldest = this.matrixRTCSession.getOldestMembership(); if (oldest !== undefined) { const selection = oldest.getTransport(oldest); - if (isLivekitTransport(selection)) local = ready(selection); + // TODO selection can be null if no transport is configured should we report an error? + if (selection && isLivekitTransport(selection)) + local = ready(selection); } } return { local, remote }; @@ -721,8 +723,8 @@ export class CallViewModel extends ViewModel { ), ); - private readonly userId = this.matrixRoom.client.getUserId(); - private readonly deviceId = this.matrixRoom.client.getDeviceId(); + private readonly userId = this.matrixRoom.client.getUserId()!; + private readonly deviceId = this.matrixRoom.client.getDeviceId()!; private readonly matrixConnected$ = this.scope.behavior( // To consider ourselves connected to MatrixRTC, we check the following: @@ -906,7 +908,11 @@ export class CallViewModel extends ViewModel { ], (memberships, _displaynames) => { const displaynameMap = new Map([ - ["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName], + [ + "local", + this.matrixRoom.getMember(this.userId)?.rawDisplayName ?? + this.userId, + ], ]); const room = this.matrixRoom; diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 785cbe1b..1b4d0cf0 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type CallMembership, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, of } from "rxjs"; import { vitest } from "vitest"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; @@ -99,12 +96,12 @@ export function getBasicRTCSession( initialRtcMemberships, ); - const rtcSession = new MockRTCSession(matrixRoom).withMemberships( + const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( rtcMemberships$, ); return { - rtcSession, + rtcSession: fakeRtcSession, matrixRoom, rtcMemberships$, }; @@ -137,7 +134,7 @@ export function getBasicCallViewModelEnvironment( // const remoteParticipants$ = of([aliceParticipant]); const vm = new CallViewModel( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), matrixRoom, mockMediaDevices({}), mockMuteStates(), diff --git a/src/utils/test.ts b/src/utils/test.ts index 508559c2..cc057532 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, vi, vitest } from "vitest"; +import { expect, type MockedObject, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -23,6 +23,7 @@ import { type SessionMembershipData, Status, type LivekitFocusSelection, + type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { @@ -193,7 +194,9 @@ export function mockRtcMembership( sender: typeof user === "string" ? user : user.userId, event_id: `$-ev-${randomUUID()}:example.org`, }); - return new CallMembership(event, data); + const cms = new CallMembership(event, data); + vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); + return cms; } // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are @@ -209,6 +212,7 @@ export function mockMatrixRoomMember( getMxcAvatarUrl(): string | undefined { return undefined; }, + rawDisplayName: rtcMembership.sender, ...member, } as RoomMember; } @@ -335,6 +339,22 @@ export class MockRTCSession extends TypedEventEmitter< RoomAndToDeviceEventsHandlerMap & MembershipManagerEventHandlerMap > { + public asMockedSession(): MockedObject { + const session = this as unknown as MockedObject; + + vi.mocked(session).reemitEncryptionKeys = vi + .fn<() => void>() + .mockReturnValue(undefined); + vi.mocked(session).resolveActiveFocus = vi + .fn<(member?: CallMembership) => Transport | undefined>() + .mockReturnValue(undefined); + vi.mocked(session).getOldestMembership = vi + .fn<() => CallMembership | undefined>() + .mockReturnValue(this.memberships[0]); + + return session; + } + public readonly statistics = { counters: {}, }; From 1ab081d6366f521532aaf2707bae3385cde2ee0d Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 10 Oct 2025 11:41:26 +0200 Subject: [PATCH 091/144] test: MISSING_MATRIX_RTC_FOCUS renamed as MISSING_MATRIX_RTC_TRANSPORT --- src/room/GroupCallErrorBoundary.test.tsx | 2 +- src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/room/GroupCallErrorBoundary.test.tsx b/src/room/GroupCallErrorBoundary.test.tsx index 22338924..86921710 100644 --- a/src/room/GroupCallErrorBoundary.test.tsx +++ b/src/room/GroupCallErrorBoundary.test.tsx @@ -106,7 +106,7 @@ test("should render the error page with link back to home", async () => { await screen.findByText("Call is not supported"); expect(screen.getByText(/Domain: example\.com/i)).toBeInTheDocument(); expect( - screen.getByText(/Error Code: MISSING_MATRIX_RTC_FOCUS/i), + screen.getByText(/Error Code: MISSING_MATRIX_RTC_TRANSPORT/i), ).toBeInTheDocument(); await screen.findByRole("button", { name: "Return to home screen" }); diff --git a/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap b/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap index ad4aff61..73a6df12 100644 --- a/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap +++ b/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap @@ -292,7 +292,7 @@ exports[`should have a close button in widget mode 1`] = ` Call is not supported

- The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS). + The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).

+ {focusUrl && ( +
+ +  {extractDomain(focusUrl)} + +
+ )} {audio && (
; - }; - - const user = userEvent.setup(); - render( - - - - - , - ); - await user.click(screen.getByRole("button", { name: "Connect" })); - screen.getByText("Insufficient capacity"); - }, -); - -describe("Leaking connection prevention", () => { - function createTestComponent(mockRoom: Room): FC { - const TestComponent: FC = () => { - const [sfuConfig, setSfuConfig] = useState( - undefined, - ); - const connect = useCallback( - () => setSfuConfig({ url: "URL", jwt: "JWT token" }), - [], - ); - useECConnectionState("default", false, mockRoom, sfuConfig); - return ; - }; - return TestComponent; - } - - test("Should cancel pending connections when the component is unmounted", async () => { - const connectCall = vi.fn(); - const pendingConnection = Promise.withResolvers(); - // let pendingDisconnection = Promise.withResolvers() - const disconnectMock = vi.fn(); - - const mockRoom = { - on: () => {}, - off: () => {}, - once: () => {}, - connect: async () => { - connectCall.call(undefined); - return await pendingConnection.promise; - }, - disconnect: disconnectMock, - localParticipant: { - getTrackPublication: () => {}, - createTracks: () => [], - }, - } as unknown as Room; - - const TestComponent = createTestComponent(mockRoom); - - const { unmount } = render(); - const user = userEvent.setup(); - await user.click(screen.getByRole("button", { name: "Connect" })); - - expect(connectCall).toHaveBeenCalled(); - // unmount while the connection is pending - unmount(); - - // resolve the pending connection - pendingConnection.resolve(); - - await vitest.waitUntil( - () => { - return disconnectMock.mock.calls.length > 0; - }, - { - timeout: 1000, - interval: 100, - }, - ); - - // There should be some cleaning up to avoid leaking an open connection - expect(disconnectMock).toHaveBeenCalledTimes(1); - }); - - test("Should cancel about to open but not yet opened connection", async () => { - const createTracksCall = vi.fn(); - const pendingCreateTrack = Promise.withResolvers(); - // let pendingDisconnection = Promise.withResolvers() - const disconnectMock = vi.fn(); - const connectMock = vi.fn(); - - const mockRoom = { - on: () => {}, - off: () => {}, - once: () => {}, - connect: connectMock, - disconnect: disconnectMock, - localParticipant: { - getTrackPublication: () => {}, - createTracks: async () => { - createTracksCall.call(undefined); - await pendingCreateTrack.promise; - return []; - }, - }, - } as unknown as Room; - - const TestComponent = createTestComponent(mockRoom); - - const { unmount } = render(); - const user = userEvent.setup(); - await user.click(screen.getByRole("button", { name: "Connect" })); - - expect(createTracksCall).toHaveBeenCalled(); - // unmount while createTracks is pending - unmount(); - - // resolve createTracks - pendingCreateTrack.resolve(); - - // Yield to the event loop to let the connection attempt finish - await sleep(100); - - // The operation should have been aborted before even calling connect. - expect(connectMock).not.toHaveBeenCalled(); - }); -}); -*/ diff --git a/src/room/MuteStates.test.tsx b/src/room/MuteStates.test.tsx deleted file mode 100644 index 530b5050..00000000 --- a/src/room/MuteStates.test.tsx +++ /dev/null @@ -1,325 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -// TODO-MULTI-SFU: These tests need to be ported to the new MuteStates class. -/* - -import { - afterAll, - afterEach, - beforeEach, - describe, - expect, - it, - onTestFinished, - vi, -} from "vitest"; -import { type FC, useCallback, useState } from "react"; -import { render, screen } from "@testing-library/react"; -import { MemoryRouter } from "react-router-dom"; -import userEvent from "@testing-library/user-event"; -import { createMediaDeviceObserver } from "@livekit/components-core"; -import { of } from "rxjs"; - -import { useMuteStates } from "./MuteStates"; -import { MediaDevicesContext } from "../MediaDevicesContext"; -import { mockConfig } from "../utils/test"; -import { MediaDevices } from "../state/MediaDevices"; -import { ObservableScope } from "../state/ObservableScope"; -vi.mock("@livekit/components-core"); - -interface TestComponentProps { - isJoined?: boolean; -} - -const TestComponent: FC = ({ isJoined = false }) => { - const muteStates = useMuteStates(isJoined); - const onToggleAudio = useCallback( - () => muteStates.audio.setEnabled?.(!muteStates.audio.enabled), - [muteStates], - ); - return ( -
-
- {muteStates.audio.enabled.toString()} -
- -
- {muteStates.video.enabled.toString()} -
-
- ); -}; - -const mockMicrophone: MediaDeviceInfo = { - deviceId: "", - kind: "audioinput", - label: "", - groupId: "", - toJSON() { - return {}; - }, -}; - -const mockSpeaker: MediaDeviceInfo = { - deviceId: "", - kind: "audiooutput", - label: "", - groupId: "", - toJSON() { - return {}; - }, -}; - -const mockCamera: MediaDeviceInfo = { - deviceId: "", - kind: "videoinput", - label: "", - groupId: "", - toJSON() { - return {}; - }, -}; - -function mockMediaDevices( - { - microphone, - speaker, - camera, - }: { - microphone?: boolean; - speaker?: boolean; - camera?: boolean; - } = { microphone: true, speaker: true, camera: true }, -): MediaDevices { - vi.mocked(createMediaDeviceObserver).mockImplementation((kind) => { - switch (kind) { - case "audioinput": - return of(microphone ? [mockMicrophone] : []); - case "audiooutput": - return of(speaker ? [mockSpeaker] : []); - case "videoinput": - return of(camera ? [mockCamera] : []); - case undefined: - throw new Error("Unimplemented"); - } - }); - return new MediaDevices(testScope()); -} - -describe("useMuteStates VITE_PACKAGE='full' (SPA) mode", () => { - afterEach(() => { - vi.clearAllMocks(); - vi.stubEnv("VITE_PACKAGE", "full"); - }); - - afterAll(() => { - vi.resetAllMocks(); - }); - - it("disabled when no input devices", () => { - mockConfig(); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("false"); - }); - - it("enables devices by default in the lobby", () => { - mockConfig(); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("true"); - expect(screen.getByTestId("video-enabled").textContent).toBe("true"); - }); - - it("disables devices by default in the call", () => { - // Disabling new devices in the call ensures that connecting a webcam - // mid-call won't cause it to suddenly be enabled without user input - mockConfig(); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("false"); - }); - - it("uses defaults from config", () => { - mockConfig({ - media_devices: { - enable_audio: false, - enable_video: false, - }, - }); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("false"); - }); - - it("skipLobby mutes inputs", () => { - mockConfig(); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("false"); - }); - - it("remembers previous state when devices disappear and reappear", async () => { - const user = userEvent.setup(); - mockConfig(); - const noDevices = mockMediaDevices({ microphone: false, camera: false }); - // Warm up these Observables before making further changes to the - // createMediaDevicesObserver mock - noDevices.audioInput.available$.subscribe(() => {}).unsubscribe(); - noDevices.videoInput.available$.subscribe(() => {}).unsubscribe(); - const someDevices = mockMediaDevices(); - - const ReappearanceTest: FC = () => { - const [devices, setDevices] = useState(someDevices); - const onConnectDevicesClick = useCallback( - () => setDevices(someDevices), - [], - ); - const onDisconnectDevicesClick = useCallback( - () => setDevices(noDevices), - [], - ); - - return ( - - - - - - - - ); - }; - - render(); - expect(screen.getByTestId("audio-enabled").textContent).toBe("true"); - expect(screen.getByTestId("video-enabled").textContent).toBe("true"); - await user.click(screen.getByRole("button", { name: "Toggle audio" })); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("true"); - await user.click( - screen.getByRole("button", { name: "Disconnect devices" }), - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("false"); - await user.click(screen.getByRole("button", { name: "Connect devices" })); - // Audio should remember that it was muted, while video should re-enable - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("true"); - }); -}); - -describe("useMuteStates in VITE_PACKAGE='embedded' (widget) mode", () => { - beforeEach(() => { - vi.stubEnv("VITE_PACKAGE", "embedded"); - }); - - it("uses defaults from config", () => { - mockConfig({ - media_devices: { - enable_audio: false, - enable_video: false, - }, - }); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("false"); - expect(screen.getByTestId("video-enabled").textContent).toBe("false"); - }); - - it("skipLobby does not mute inputs", () => { - mockConfig(); - - render( - - - - - , - ); - expect(screen.getByTestId("audio-enabled").textContent).toBe("true"); - expect(screen.getByTestId("video-enabled").textContent).toBe("true"); - }); - - it("url params win over config", () => { - // The config sets audio and video to disabled - mockConfig({ media_devices: { enable_audio: false, enable_video: false } }); - - render( - - - - - , - ); - // At the end we expect the url param to take precedence, resulting in true - expect(screen.getByTestId("audio-enabled").textContent).toBe("true"); - expect(screen.getByTestId("video-enabled").textContent).toBe("true"); - }); -}); -*/ From 6be774909162e42c80509df5e0ecb356bdaa3080 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 22 Oct 2025 23:27:38 -0400 Subject: [PATCH 132/144] Remove even more dead code --- src/room/GroupCallView.test.tsx | 1 + src/room/checkForParallelCalls.test.ts | 155 ------------------------- src/room/checkForParallelCalls.ts | 55 --------- src/rtcSessionHelpers.test.ts | 46 +------- src/rtcSessionHelpers.ts | 48 +------- src/useErrorBoundary.test.tsx | 51 -------- src/useErrorBoundary.ts | 29 ----- 7 files changed, 4 insertions(+), 381 deletions(-) delete mode 100644 src/room/checkForParallelCalls.test.ts delete mode 100644 src/room/checkForParallelCalls.ts delete mode 100644 src/useErrorBoundary.test.tsx delete mode 100644 src/useErrorBoundary.ts diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 10c55b5b..ad884865 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -81,6 +81,7 @@ vi.mock("../rtcSessionHelpers", async (importOriginal) => { // TODO: perhaps there is a more elegant way to manage the type import here? // eslint-disable-next-line @typescript-eslint/consistent-type-imports const orig = await importOriginal(); + // TODO: leaveRTCSession no longer exists! Tests need adapting. return { ...orig, enterRTCSession, leaveRTCSession }; }); diff --git a/src/room/checkForParallelCalls.test.ts b/src/room/checkForParallelCalls.test.ts deleted file mode 100644 index 2d14c79a..00000000 --- a/src/room/checkForParallelCalls.test.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { vi, type Mocked, test, expect } from "vitest"; -import { type RoomState } from "matrix-js-sdk"; - -import { PosthogAnalytics } from "../../src/analytics/PosthogAnalytics"; -import { checkForParallelCalls } from "../../src/room/checkForParallelCalls"; -import { withFakeTimers } from "../utils/test"; - -const withMockedPosthog = ( - continuation: (posthog: Mocked) => void, -): void => { - const posthog = vi.mocked({ - trackEvent: vi.fn(), - } as unknown as PosthogAnalytics); - const instanceSpy = vi - .spyOn(PosthogAnalytics, "instance", "get") - .mockReturnValue(posthog); - try { - continuation(posthog); - } finally { - instanceSpy.mockRestore(); - } -}; - -const mockRoomState = ( - groupCallMemberContents: Record[], -): RoomState => { - const stateEvents = groupCallMemberContents.map((content) => ({ - getContent: (): Record => content, - })); - return { getStateEvents: () => stateEvents } as unknown as RoomState; -}; - -test("checkForParallelCalls does nothing if all participants are in the same call", () => { - withFakeTimers(() => { - withMockedPosthog((posthog) => { - const roomState = mockRoomState([ - { - "m.calls": [ - { - "m.call_id": "1", - "m.devices": [ - { - device_id: "Element Call", - session_id: "a", - expires_ts: Date.now() + 1000, - }, - ], - }, - { - "m.call_id": null, // invalid - "m.devices": [ - { - device_id: "Element Android", - session_id: "a", - expires_ts: Date.now() + 1000, - }, - ], - }, - null, // invalid - ], - }, - { - "m.calls": [ - { - "m.call_id": "1", - "m.devices": [ - { - device_id: "Element Desktop", - session_id: "a", - expires_ts: Date.now() + 1000, - }, - ], - }, - ], - }, - ]); - - checkForParallelCalls(roomState); - expect(posthog.trackEvent).not.toHaveBeenCalled(); - }); - }); -}); - -test("checkForParallelCalls sends diagnostics to PostHog if there is a split-brain", () => { - withFakeTimers(() => { - withMockedPosthog((posthog) => { - const roomState = mockRoomState([ - { - "m.calls": [ - { - "m.call_id": "1", - "m.devices": [ - { - device_id: "Element Call", - session_id: "a", - expires_ts: Date.now() + 1000, - }, - ], - }, - { - "m.call_id": "2", - "m.devices": [ - { - device_id: "Element Android", - session_id: "a", - expires_ts: Date.now() + 1000, - }, - ], - }, - ], - }, - { - "m.calls": [ - { - "m.call_id": "1", - "m.devices": [ - { - device_id: "Element Desktop", - session_id: "a", - expires_ts: Date.now() + 1000, - }, - ], - }, - { - "m.call_id": "2", - "m.devices": [ - { - device_id: "Element Call", - session_id: "a", - expires_ts: Date.now() - 1000, - }, - ], - }, - ], - }, - ]); - - checkForParallelCalls(roomState); - expect(posthog.trackEvent).toHaveBeenCalledWith({ - eventName: "ParallelCalls", - participantsPerCall: { - "1": 2, - "2": 1, - }, - }); - }); - }); -}); diff --git a/src/room/checkForParallelCalls.ts b/src/room/checkForParallelCalls.ts deleted file mode 100644 index ab947176..00000000 --- a/src/room/checkForParallelCalls.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { EventType, type RoomState } from "matrix-js-sdk"; - -import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; - -function isObject(x: unknown): x is Record { - return typeof x === "object" && x !== null; -} - -/** - * Checks the state of a room for multiple calls happening in parallel, sending - * the details to PostHog if that is indeed what's happening. (This is unwanted - * as it indicates a split-brain scenario.) - */ -export function checkForParallelCalls(state: RoomState): void { - const now = Date.now(); - const participantsPerCall = new Map(); - - // For each participant in each call, increment the participant count - for (const e of state.getStateEvents(EventType.GroupCallMemberPrefix)) { - const content = e.getContent>(); - const calls: unknown[] = Array.isArray(content["m.calls"]) - ? content["m.calls"] - : []; - - for (const call of calls) { - if (isObject(call) && typeof call["m.call_id"] === "string") { - const devices: unknown[] = Array.isArray(call["m.devices"]) - ? call["m.devices"] - : []; - - for (const device of devices) { - if (isObject(device) && (device["expires_ts"] as number) > now) { - const participantCount = - participantsPerCall.get(call["m.call_id"]) ?? 0; - participantsPerCall.set(call["m.call_id"], participantCount + 1); - } - } - } - } - } - - if (participantsPerCall.size > 1) { - PosthogAnalytics.instance.trackEvent({ - eventName: "ParallelCalls", - participantsPerCall: Object.fromEntries(participantsPerCall), - }); - } -} diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index e6b76784..8aca40f5 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -6,13 +6,12 @@ Please see LICENSE in the repository root for full details. */ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { expect, onTestFinished, test, vi } from "vitest"; +import { expect, test, vi } from "vitest"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import EventEmitter from "events"; -import { enterRTCSession, leaveRTCSession } from "../src/rtcSessionHelpers"; +import { enterRTCSession } from "../src/rtcSessionHelpers"; import { mockConfig } from "./utils/test"; -import { ElementWidgetActions, widget } from "./widget"; const USE_MUTI_SFU = false; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); @@ -116,47 +115,6 @@ test("It joins the correct Session", async () => { ); }); -async function testLeaveRTCSession( - cause: "user" | "error", - expectClose: boolean, -): Promise { - vi.clearAllMocks(); - const session = { leaveRoomSession: vi.fn() } as unknown as MatrixRTCSession; - await leaveRTCSession(session, cause); - expect(session.leaveRoomSession).toHaveBeenCalled(); - expect(widget!.api.transport.send).toHaveBeenCalledWith( - ElementWidgetActions.HangupCall, - expect.anything(), - ); - if (expectClose) { - expect(widget!.api.transport.send).toHaveBeenCalledWith( - ElementWidgetActions.Close, - expect.anything(), - ); - expect(widget!.api.transport.stop).toHaveBeenCalled(); - } else { - expect(widget!.api.transport.send).not.toHaveBeenCalledWith( - ElementWidgetActions.Close, - expect.anything(), - ); - expect(widget!.api.transport.stop).not.toHaveBeenCalled(); - } -} - -test("leaveRTCSession closes the widget on a normal hangup", async () => { - await testLeaveRTCSession("user", true); -}); - -test("leaveRTCSession doesn't close the widget on a fatal error", async () => { - await testLeaveRTCSession("error", false); -}); - -test("leaveRTCSession doesn't close the widget when returning to lobby", async () => { - getUrlParams.mockReturnValue({ returnToLobby: true }); - onTestFinished(() => void getUrlParams.mockReset()); - await testLeaveRTCSession("user", false); -}); - test("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => { mockConfig({}); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({ diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 90e1fca1..fadc7b37 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -16,7 +16,7 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; import { Config } from "./config/Config"; -import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; +import { ElementWidgetActions, widget } from "./widget"; import { MatrixRTCTransportMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; @@ -159,49 +159,3 @@ export async function enterRTCSession( } } } - -const widgetPostHangupProcedure = async ( - widget: WidgetHelpers, - cause: "user" | "error", - promiseBeforeHangup?: Promise, -): Promise => { - try { - await widget.api.setAlwaysOnScreen(false); - } catch (e) { - logger.error("Failed to set call widget `alwaysOnScreen` to false", e); - } - - // Wait for any last bits before hanging up. - await promiseBeforeHangup; - // We send the hangup event after the memberships have been updated - // calling leaveRTCSession. - // We need to wait because this makes the client hosting this widget killing the IFrame. - try { - await widget.api.transport.send(ElementWidgetActions.HangupCall, {}); - } catch (e) { - logger.error("Failed to send hangup action", e); - } - // On a normal user hangup we can shut down and close the widget. But if an - // error occurs we should keep the widget open until the user reads it. - if (cause === "user" && !getUrlParams().returnToLobby) { - try { - await widget.api.transport.send(ElementWidgetActions.Close, {}); - } catch (e) { - logger.error("Failed to send close action", e); - } - widget.api.transport.stop(); - } -}; - -export async function leaveRTCSession( - rtcSession: MatrixRTCSession, - cause: "user" | "error", - promiseBeforeHangup?: Promise, -): Promise { - await rtcSession.leaveRoomSession(); - if (widget) { - await widgetPostHangupProcedure(widget, cause, promiseBeforeHangup); - } else { - await promiseBeforeHangup; - } -} diff --git a/src/useErrorBoundary.test.tsx b/src/useErrorBoundary.test.tsx deleted file mode 100644 index 13fa43bb..00000000 --- a/src/useErrorBoundary.test.tsx +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { it, vi } from "vitest"; -import { render, screen } from "@testing-library/react"; -import { type ReactElement, useCallback } from "react"; -import userEvent from "@testing-library/user-event"; -import { BrowserRouter } from "react-router-dom"; - -import { GroupCallErrorBoundary } from "./room/GroupCallErrorBoundary"; -import { useErrorBoundary } from "./useErrorBoundary"; -import { ConnectionLostError } from "./utils/errors"; - -it("should show async error", async () => { - const user = userEvent.setup(); - - const TestComponent = (): ReactElement => { - const { showErrorBoundary } = useErrorBoundary(); - - const onClick = useCallback((): void => { - showErrorBoundary(new ConnectionLostError()); - }, [showErrorBoundary]); - - return ( -
-

HELLO

- -
- ); - }; - - render( - - - - - , - ); - - await user.click(screen.getByRole("button", { name: "Click me" })); - - await screen.findByText("Connection lost"); - - await user.click(screen.getByRole("button", { name: "Reconnect" })); - - await screen.findByText("HELLO"); -}); diff --git a/src/useErrorBoundary.ts b/src/useErrorBoundary.ts deleted file mode 100644 index 4430394e..00000000 --- a/src/useErrorBoundary.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { useMemo, useState } from "react"; - -export type UseErrorBoundaryApi = { - showErrorBoundary: (error: Error) => void; -}; - -export function useErrorBoundary(): UseErrorBoundaryApi { - const [error, setError] = useState(null); - - const memoized: UseErrorBoundaryApi = useMemo( - () => ({ - showErrorBoundary: (error: Error) => setError(error), - }), - [], - ); - - if (error) { - throw error; - } - - return memoized; -} From db2004fb90cd68cfa12ae9d1fc446586bb4667f3 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 22 Oct 2025 23:29:36 -0400 Subject: [PATCH 133/144] Remove unused string --- locales/en/app.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 71b087ac..11267439 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -79,8 +79,7 @@ "label": "Prefer sticky events" }, "show_connection_stats": "Show connection statistics", - "url_params": "URL parameters", - "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key" + "url_params": "URL parameters" }, "disconnected_banner": "Connectivity to the server has been lost.", "error": { From e06f288e935be7ffe51f015849144488f28d5f0d Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 10:41:10 +0200 Subject: [PATCH 134/144] update playwright --- package.json | 2 +- yarn.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index cbcc5d03..d7e64d95 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/sdk-trace-web": "^2.0.0", "@opentelemetry/semantic-conventions": "^1.25.1", - "@playwright/test": "^1.52.0", + "@playwright/test": "^1.56.1", "@radix-ui/react-dialog": "^1.0.4", "@radix-ui/react-slider": "^1.1.2", "@radix-ui/react-visually-hidden": "^1.0.3", diff --git a/yarn.lock b/yarn.lock index d9b9864f..2f60af8f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3371,14 +3371,14 @@ __metadata: languageName: node linkType: hard -"@playwright/test@npm:^1.52.0": - version: 1.54.1 - resolution: "@playwright/test@npm:1.54.1" +"@playwright/test@npm:^1.56.1": + version: 1.56.1 + resolution: "@playwright/test@npm:1.56.1" dependencies: - playwright: "npm:1.54.1" + playwright: "npm:1.56.1" bin: playwright: cli.js - checksum: 10c0/1b414356bc1049927d7b9efc14d5b3bf000ef6483313926bb795b4f27fe3707e8e0acf0db59063a452bb4f7e34559758d17640401b6f3e2f5290f299a8d8d02f + checksum: 10c0/2b5b0e1f2e6a18f6e5ce6897c7440ca78f64e0b004834e9808e93ad2b78b96366b562ae4366602669cf8ad793a43d85481b58541e74be71e905e732d833dd691 languageName: node linkType: hard @@ -7490,7 +7490,7 @@ __metadata: "@opentelemetry/sdk-trace-base": "npm:^2.0.0" "@opentelemetry/sdk-trace-web": "npm:^2.0.0" "@opentelemetry/semantic-conventions": "npm:^1.25.1" - "@playwright/test": "npm:^1.52.0" + "@playwright/test": "npm:^1.56.1" "@radix-ui/react-dialog": "npm:^1.0.4" "@radix-ui/react-slider": "npm:^1.1.2" "@radix-ui/react-visually-hidden": "npm:^1.0.3" @@ -11160,27 +11160,27 @@ __metadata: languageName: node linkType: hard -"playwright-core@npm:1.54.1": - version: 1.54.1 - resolution: "playwright-core@npm:1.54.1" +"playwright-core@npm:1.56.1": + version: 1.56.1 + resolution: "playwright-core@npm:1.56.1" bin: playwright-core: cli.js - checksum: 10c0/b821262b024d7753b1bfa71eb2bc99f2dda12a869d175b2e1bc6ac2764bd661baf36d9d42f45caf622854ad7e4a6077b9b57014c74bb5a78fe339c9edf1c9019 + checksum: 10c0/ffd40142b99c68678b387445d5b42f1fee4ab0b65d983058c37f342e5629f9cdbdac0506ea80a0dfd41a8f9f13345bad54e9a8c35826ef66dc765f4eb3db8da7 languageName: node linkType: hard -"playwright@npm:1.54.1": - version: 1.54.1 - resolution: "playwright@npm:1.54.1" +"playwright@npm:1.56.1": + version: 1.56.1 + resolution: "playwright@npm:1.56.1" dependencies: fsevents: "npm:2.3.2" - playwright-core: "npm:1.54.1" + playwright-core: "npm:1.56.1" dependenciesMeta: fsevents: optional: true bin: playwright: cli.js - checksum: 10c0/c5fedae31a03a1f4c4846569aef3ffb98da23000a4d255abfc8c2ede15b43cc7cd87b80f6fa078666c030373de8103787cf77ef7653ae9458aabbbd4320c2599 + checksum: 10c0/8e9965aede86df0f4722063385748498977b219630a40a10d1b82b8bd8d4d4e9b6b65ecbfa024331a30800163161aca292fb6dd7446c531a1ad25f4155625ab4 languageName: node linkType: hard From a638acde2c58e2b744dad4d0f7ca90a0112ce7de Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 13:41:08 +0200 Subject: [PATCH 135/144] CI: Increase job timeout --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4fd2dcd0..54035ea4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -30,7 +30,7 @@ jobs: fail_ci_if_error: true playwright: name: Run end-to-end tests - timeout-minutes: 30 + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 From 348a6fe1af7e67693dfafb070882f5ba6e1af42d Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 13:54:08 +0200 Subject: [PATCH 136/144] test CI changing ports --- playwright/fixtures/widget-user.ts | 2 +- scripts/playwright-webserver-command.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index 8089c9de..484c1712 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -76,7 +76,7 @@ const setDevToolElementCallDevUrl = process.env.USE_DOCKER "Developer.elementCallUrl", null, "device", - "http://localhost:8080/room", + "http://localhost:3000/room", ); }); } diff --git a/scripts/playwright-webserver-command.sh b/scripts/playwright-webserver-command.sh index 8c00909b..c6015fef 100755 --- a/scripts/playwright-webserver-command.sh +++ b/scripts/playwright-webserver-command.sh @@ -3,7 +3,7 @@ if [ -n "$USE_DOCKER" ]; then set -ex yarn build docker build -t element-call:testing . - exec docker run --rm --name element-call-testing -p 8080:8080 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing + exec docker run --rm --name element-call-testing -p 3000:3000 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing else cp config/config.devenv.json public/config.json exec yarn dev From c572d2e7794018dc8836c31630e2d4089f0928e2 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 14:02:17 +0200 Subject: [PATCH 137/144] test CI --- playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright.config.ts b/playwright.config.ts index 7a8ee530..8dfcb8e7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { defineConfig, devices } from "@playwright/test"; const baseURL = process.env.USE_DOCKER - ? "http://localhost:8080" + ? "http://localhost:3000" : "https://localhost:3000"; /** From 185d7d117758545315954b820936e36fe9ad868a Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 14:13:50 +0200 Subject: [PATCH 138/144] CI test --- playwright.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/playwright.config.ts b/playwright.config.ts index 8dfcb8e7..dd7b1e76 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -81,6 +81,7 @@ export default defineConfig({ url: baseURL, reuseExistingServer: !process.env.CI, ignoreHTTPSErrors: true, + timeout: 240000, gracefulShutdown: { signal: "SIGTERM", timeout: 500, From 552e1215befcf5bbfe2472dfa2a8cd53af3db7e6 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 14:24:15 +0200 Subject: [PATCH 139/144] revert CI test changes --- playwright.config.ts | 3 +-- playwright/fixtures/widget-user.ts | 2 +- scripts/playwright-webserver-command.sh | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index dd7b1e76..7a8ee530 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { defineConfig, devices } from "@playwright/test"; const baseURL = process.env.USE_DOCKER - ? "http://localhost:3000" + ? "http://localhost:8080" : "https://localhost:3000"; /** @@ -81,7 +81,6 @@ export default defineConfig({ url: baseURL, reuseExistingServer: !process.env.CI, ignoreHTTPSErrors: true, - timeout: 240000, gracefulShutdown: { signal: "SIGTERM", timeout: 500, diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index 484c1712..8089c9de 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -76,7 +76,7 @@ const setDevToolElementCallDevUrl = process.env.USE_DOCKER "Developer.elementCallUrl", null, "device", - "http://localhost:3000/room", + "http://localhost:8080/room", ); }); } diff --git a/scripts/playwright-webserver-command.sh b/scripts/playwright-webserver-command.sh index c6015fef..8c00909b 100755 --- a/scripts/playwright-webserver-command.sh +++ b/scripts/playwright-webserver-command.sh @@ -3,7 +3,7 @@ if [ -n "$USE_DOCKER" ]; then set -ex yarn build docker build -t element-call:testing . - exec docker run --rm --name element-call-testing -p 3000:3000 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing + exec docker run --rm --name element-call-testing -p 8080:8080 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing else cp config/config.devenv.json public/config.json exec yarn dev From 150cdf64c82de31c7d796c60a535f4d4dcdd9700 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 15:55:22 +0200 Subject: [PATCH 140/144] CI try some more logs --- .github/workflows/test.yaml | 9 +++++++-- scripts/playwright-webserver-command.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 54035ea4..708af235 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,16 +43,21 @@ jobs: - name: Install dependencies run: yarn install --immutable - name: Install Playwright Browsers - run: yarn playwright install --with-deps + run: | + echo "Installing Playwright browsers..." + yarn playwright install --with-deps - name: Run backend components run: | + echo "Run backend components..." docker compose -f playwright-backend-docker-compose.yml -f playwright-backend-docker-compose.override.yml pull docker compose -f playwright-backend-docker-compose.yml -f playwright-backend-docker-compose.override.yml up -d docker ps - name: Run Playwright tests env: USE_DOCKER: 1 - run: yarn playwright test + run: | + echo "Run playwright tests..." + yarn playwright test - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: ${{ !cancelled() }} with: diff --git a/scripts/playwright-webserver-command.sh b/scripts/playwright-webserver-command.sh index 8c00909b..1bea6dcf 100755 --- a/scripts/playwright-webserver-command.sh +++ b/scripts/playwright-webserver-command.sh @@ -3,7 +3,7 @@ if [ -n "$USE_DOCKER" ]; then set -ex yarn build docker build -t element-call:testing . - exec docker run --rm --name element-call-testing -p 8080:8080 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing + exec docker run --log-level debug --rm --name element-call-testing -p 8080:8080 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing else cp config/config.devenv.json public/config.json exec yarn dev From 0823936ee1fd091c6356fc0eda3eb0fc83ad628f Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 16:05:24 +0200 Subject: [PATCH 141/144] revert CI test changes --- .github/workflows/test.yaml | 9 ++------- scripts/playwright-webserver-command.sh | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 708af235..54035ea4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,21 +43,16 @@ jobs: - name: Install dependencies run: yarn install --immutable - name: Install Playwright Browsers - run: | - echo "Installing Playwright browsers..." - yarn playwright install --with-deps + run: yarn playwright install --with-deps - name: Run backend components run: | - echo "Run backend components..." docker compose -f playwright-backend-docker-compose.yml -f playwright-backend-docker-compose.override.yml pull docker compose -f playwright-backend-docker-compose.yml -f playwright-backend-docker-compose.override.yml up -d docker ps - name: Run Playwright tests env: USE_DOCKER: 1 - run: | - echo "Run playwright tests..." - yarn playwright test + run: yarn playwright test - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: ${{ !cancelled() }} with: diff --git a/scripts/playwright-webserver-command.sh b/scripts/playwright-webserver-command.sh index 1bea6dcf..8c00909b 100755 --- a/scripts/playwright-webserver-command.sh +++ b/scripts/playwright-webserver-command.sh @@ -3,7 +3,7 @@ if [ -n "$USE_DOCKER" ]; then set -ex yarn build docker build -t element-call:testing . - exec docker run --log-level debug --rm --name element-call-testing -p 8080:8080 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing + exec docker run --rm --name element-call-testing -p 8080:8080 -v ./config/config.devenv.json:/app/config.json:ro,Z element-call:testing else cp config/config.devenv.json public/config.json exec yarn dev From d52656c5532258677db9d5cdf7d778621aa9d2fe Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 16:07:04 +0200 Subject: [PATCH 142/144] CI try without USE_DOCKER --- .github/workflows/test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 54035ea4..44c14842 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,8 +50,6 @@ jobs: docker compose -f playwright-backend-docker-compose.yml -f playwright-backend-docker-compose.override.yml up -d docker ps - name: Run Playwright tests - env: - USE_DOCKER: 1 run: yarn playwright test - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: ${{ !cancelled() }} From 918d67b2ac40567ebe9edc988360b4ac686872e5 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 16:23:08 +0200 Subject: [PATCH 143/144] CI: revert to synapse latest --- dev-backend-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index e64ba80e..50498c7a 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -88,7 +88,7 @@ services: synapse: hostname: homeserver - image: ghcr.io/element-hq/synapse:msc4354-5 + image: docker.io/matrixdotorg/synapse:latest pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml From 301cf2f4ebf78c8ce9f6b6028eb7791f0e783f79 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 23 Oct 2025 16:26:02 +0200 Subject: [PATCH 144/144] Revert "CI try without USE_DOCKER" This reverts commit d52656c5532258677db9d5cdf7d778621aa9d2fe. --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 44c14842..54035ea4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,6 +50,8 @@ jobs: docker compose -f playwright-backend-docker-compose.yml -f playwright-backend-docker-compose.override.yml up -d docker ps - name: Run Playwright tests + env: + USE_DOCKER: 1 run: yarn playwright test - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: ${{ !cancelled() }}