pull out all screen share related logic.
This commit is contained in:
Timo K
210
src/state/CallViewModel/CallNotificationLifecycle.ts
Normal file
210
src/state/CallViewModel/CallNotificationLifecycle.ts
Normal file
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CallMembership,
|
||||
type MatrixRTCSession,
|
||||
MatrixRTCSessionEvent,
|
||||
type MatrixRTCSessionEventHandlerMap,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
combineLatest,
|
||||
concat,
|
||||
endWith,
|
||||
filter,
|
||||
fromEvent,
|
||||
ignoreElements,
|
||||
map,
|
||||
merge,
|
||||
NEVER,
|
||||
type Observable,
|
||||
of,
|
||||
pairwise,
|
||||
startWith,
|
||||
switchMap,
|
||||
takeUntil,
|
||||
timer,
|
||||
} from "rxjs";
|
||||
import {
|
||||
type EventTimelineSetHandlerMap,
|
||||
EventType,
|
||||
type Room as MatrixRoom,
|
||||
RoomEvent,
|
||||
} from "matrix-js-sdk";
|
||||
|
||||
import { type Behavior } from "../Behavior";
|
||||
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
|
||||
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
|
||||
export type CallPickupState =
|
||||
| "unknown"
|
||||
| "ringing"
|
||||
| "timeout"
|
||||
| "decline"
|
||||
| "success"
|
||||
| null;
|
||||
export type CallNotificationWrapper = Parameters<
|
||||
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
|
||||
>;
|
||||
export function createSentCallNotification$(
|
||||
scope: ObservableScope,
|
||||
matrixRTCSession: MatrixRTCSession,
|
||||
): Behavior<CallNotificationWrapper | null> {
|
||||
const sentCallNotification$ = scope.behavior(
|
||||
fromEvent(matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification),
|
||||
null,
|
||||
) as Behavior<CallNotificationWrapper | null>;
|
||||
return sentCallNotification$;
|
||||
}
|
||||
|
||||
export function createReceivedDecline$(
|
||||
matrixRoom: MatrixRoom,
|
||||
): Observable<Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>> {
|
||||
return (
|
||||
fromEvent(matrixRoom, RoomEvent.Timeline) as Observable<
|
||||
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
|
||||
>
|
||||
).pipe(filter(([event]) => event.getType() === EventType.RTCDecline));
|
||||
}
|
||||
|
||||
interface Props {
|
||||
scope: ObservableScope;
|
||||
memberships$: Behavior<Epoch<CallMembership[]>>;
|
||||
sentCallNotification$: Observable<CallNotificationWrapper | null>;
|
||||
receivedDecline$: Observable<
|
||||
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
|
||||
>;
|
||||
options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean };
|
||||
localUser: { deviceId: string; userId: string };
|
||||
}
|
||||
/**
|
||||
* @returns {callPickupState$, autoLeave$}
|
||||
* `callPickupState$` The current call pickup state of the call.
|
||||
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
|
||||
* Then we can conclude if we were the first one to join or not.
|
||||
* This may also be set if we are disconnected.
|
||||
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
|
||||
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
|
||||
* The call failed. If desired this can be used as a trigger to exit the call.
|
||||
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
|
||||
* - null: EC is configured to never show any waiting for answer state.
|
||||
*
|
||||
* `autoLeave$` An observable that emits (null) when the call should be automatically left.
|
||||
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
|
||||
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
|
||||
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
|
||||
*
|
||||
*/
|
||||
export function createCallNotificationLifecycle$({
|
||||
scope,
|
||||
memberships$,
|
||||
sentCallNotification$,
|
||||
receivedDecline$,
|
||||
options,
|
||||
localUser,
|
||||
}: Props): {
|
||||
callPickupState$: Behavior<CallPickupState>;
|
||||
autoLeave$: Observable<AutoLeaveReason>;
|
||||
} {
|
||||
// TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$}
|
||||
const allOthersLeft$ = memberships$.pipe(
|
||||
pairwise(),
|
||||
filter(
|
||||
([{ value: prev }, { value: current }]) =>
|
||||
current.every((m) => m.userId === localUser.userId) &&
|
||||
prev.some((m) => m.userId !== localUser.userId),
|
||||
),
|
||||
map(() => {}),
|
||||
);
|
||||
|
||||
/**
|
||||
* Whether some Matrix user other than ourself is joined to the call.
|
||||
*/
|
||||
const someoneElseJoined$ = memberships$.pipe(
|
||||
mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
|
||||
) as Behavior<Epoch<boolean>>;
|
||||
|
||||
/**
|
||||
* Whenever the RTC session tells us that it intends to ring the remote
|
||||
* participant's devices, this emits an Observable tracking the current state of
|
||||
* that ringing process.
|
||||
*/
|
||||
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
|
||||
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
|
||||
// A behavior will emit the latest observable with the running timer to new subscribers.
|
||||
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
|
||||
// `ring$` would not be a behavior.
|
||||
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
|
||||
scope.behavior(
|
||||
sentCallNotification$.pipe(
|
||||
filter(
|
||||
(newAndLegacyEvents) =>
|
||||
// only care about new events (legacy do not have decline pattern)
|
||||
newAndLegacyEvents?.[0].notification_type === "ring",
|
||||
),
|
||||
map((e) => e as CallNotificationWrapper),
|
||||
switchMap(([notificastionEvent]) => {
|
||||
const lifetimeMs = notificationEvent?.lifetime ?? 0;
|
||||
return concat(
|
||||
lifetimeMs === 0
|
||||
? // If no lifetime, skip the ring state
|
||||
of(null)
|
||||
: // Ring until lifetime ms have passed
|
||||
timer(lifetimeMs).pipe(
|
||||
ignoreElements(),
|
||||
startWith("ringing" as const),
|
||||
),
|
||||
// The notification lifetime has timed out, meaning ringing has likely
|
||||
// stopped on all receiving clients.
|
||||
of("timeout" as const),
|
||||
// This makes sure we will not drop into the `endWith("decline" as const)` state
|
||||
NEVER,
|
||||
).pipe(
|
||||
takeUntil(
|
||||
receivedDecline$.pipe(
|
||||
filter(
|
||||
([event]) =>
|
||||
event.getRelation()?.rel_type === "m.reference" &&
|
||||
event.getRelation()?.event_id ===
|
||||
notificationEvent.event_id &&
|
||||
event.getSender() !== localUser.userId,
|
||||
),
|
||||
),
|
||||
),
|
||||
endWith("decline" as const),
|
||||
);
|
||||
}),
|
||||
),
|
||||
null,
|
||||
);
|
||||
|
||||
const callPickupState$ = scope.behavior(
|
||||
options.waitForCallPickup === true
|
||||
? combineLatest(
|
||||
[someoneElseJoined$, remoteRingState$],
|
||||
(someoneElseJoined, ring) => {
|
||||
if (someoneElseJoined) {
|
||||
return "success" as const;
|
||||
}
|
||||
// Show the ringing state of the most recent ringing attempt.
|
||||
// as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown.
|
||||
return ring ?? ("unknown" as const);
|
||||
},
|
||||
)
|
||||
: NEVER,
|
||||
null,
|
||||
);
|
||||
|
||||
const autoLeave$ = merge(
|
||||
options.autoLeaveWhenOthersLeft === true
|
||||
? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
|
||||
: NEVER,
|
||||
callPickupState$.pipe(
|
||||
filter((state) => state === "timeout" || state === "decline"),
|
||||
),
|
||||
);
|
||||
return { autoLeave$, callPickupState$ };
|
||||
}
|
||||
1233
src/state/CallViewModel/CallViewModel.ts
Normal file
1233
src/state/CallViewModel/CallViewModel.ts
Normal file
File diff suppressed because it is too large
Load Diff
0
src/state/CallViewModel/callPickupState$.ts
Normal file
0
src/state/CallViewModel/callPickupState$.ts
Normal file
508
src/state/CallViewModel/localMember/LocalMembership.ts
Normal file
508
src/state/CallViewModel/localMember/LocalMembership.ts
Normal file
@@ -0,0 +1,508 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type LocalTrack, type E2EEOptions } from "livekit-client";
|
||||
import {
|
||||
type LivekitTransport,
|
||||
type MatrixRTCSession,
|
||||
MembershipManagerEvent,
|
||||
Status,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk";
|
||||
import {
|
||||
BehaviorSubject,
|
||||
combineLatest,
|
||||
fromEvent,
|
||||
map,
|
||||
NEVER,
|
||||
type Observable,
|
||||
of,
|
||||
scan,
|
||||
startWith,
|
||||
switchMap,
|
||||
take,
|
||||
takeWhile,
|
||||
tap,
|
||||
} from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { sharingScreen$ as observeSharingScreen$ } from "../../UserMedia.ts";
|
||||
import { type Behavior } from "../../Behavior";
|
||||
import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
|
||||
import { ObservableScope } from "../../ObservableScope";
|
||||
import { Publisher } from "./Publisher";
|
||||
import { type MuteStates } from "../../MuteStates";
|
||||
import { type ProcessorState } from "../../../livekit/TrackProcessorContext";
|
||||
import { type MediaDevices } from "../../MediaDevices";
|
||||
import { and$ } from "../../../utils/observable";
|
||||
import {
|
||||
enterRTCSession,
|
||||
type EnterRTCSessionOptions,
|
||||
} from "../../../rtcSessionHelpers";
|
||||
import { type ElementCallError } from "../../../utils/errors";
|
||||
import { ElementWidgetActions, type WidgetHelpers } from "../../../widget";
|
||||
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers";
|
||||
import { getUrlParams } from "../../../UrlParams.ts";
|
||||
|
||||
export enum LivekitState {
|
||||
Uninitialized = "uninitialized",
|
||||
Connecting = "connecting",
|
||||
Connected = "connected",
|
||||
Error = "error",
|
||||
Disconnected = "disconnected",
|
||||
Disconnecting = "disconnecting",
|
||||
}
|
||||
type LocalMemberLivekitState =
|
||||
| { state: LivekitState.Error; error: string }
|
||||
| { state: LivekitState.Connected }
|
||||
| { state: LivekitState.Connecting }
|
||||
| { state: LivekitState.Uninitialized }
|
||||
| { state: LivekitState.Disconnected }
|
||||
| { state: LivekitState.Disconnecting };
|
||||
|
||||
export enum MatrixState {
|
||||
Connected = "connected",
|
||||
Disconnected = "disconnected",
|
||||
Connecting = "connecting",
|
||||
}
|
||||
type LocalMemberMatrixState =
|
||||
| { state: MatrixState.Connected }
|
||||
| { state: MatrixState.Connecting }
|
||||
| { state: MatrixState.Disconnected };
|
||||
|
||||
export interface LocalMemberConnectionState {
|
||||
livekit$: BehaviorSubject<LocalMemberLivekitState>;
|
||||
matrix$: BehaviorSubject<LocalMemberMatrixState>;
|
||||
}
|
||||
|
||||
/*
|
||||
* - get well known
|
||||
* - get oldest membership
|
||||
* - get transport to use
|
||||
* - get openId + jwt token
|
||||
* - wait for createTrack() call
|
||||
* - create tracks
|
||||
* - wait for join() call
|
||||
* - Publisher.publishTracks()
|
||||
* - send join state/sticky event
|
||||
*/
|
||||
interface Props {
|
||||
options: Behavior<EnterRTCSessionOptions>;
|
||||
scope: ObservableScope;
|
||||
mediaDevices: MediaDevices;
|
||||
muteStates: MuteStates;
|
||||
connectionManager: IConnectionManager;
|
||||
matrixRTCSession: MatrixRTCSession;
|
||||
matrixRoom: MatrixRoom;
|
||||
localTransport$: Behavior<LivekitTransport | undefined>;
|
||||
e2eeLivekitOptions: E2EEOptions | undefined;
|
||||
trackProcessorState$: Behavior<ProcessorState>;
|
||||
widget: WidgetHelpers | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* This class is responsible for managing the own membership in a room.
|
||||
* We want
|
||||
* - a publisher
|
||||
* -
|
||||
* @param param0
|
||||
* @returns
|
||||
* - publisher: The handle to create tracks and publish them to the room.
|
||||
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
|
||||
* - transport$: the transport object the ownMembership$ ended up using.
|
||||
* - connectionState: the current connection state. Including matrix server and livekit server connection.
|
||||
* - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
|
||||
*/
|
||||
export const createLocalMembership$ = ({
|
||||
scope,
|
||||
options,
|
||||
muteStates,
|
||||
mediaDevices,
|
||||
connectionManager,
|
||||
matrixRTCSession,
|
||||
localTransport$,
|
||||
matrixRoom,
|
||||
e2eeLivekitOptions,
|
||||
trackProcessorState$,
|
||||
widget,
|
||||
}: Props): {
|
||||
// publisher: Publisher
|
||||
requestConnect: () => LocalMemberConnectionState;
|
||||
startTracks: () => Behavior<LocalTrack[]>;
|
||||
requestDisconnect: () => Observable<LocalMemberLivekitState> | null;
|
||||
connectionState: LocalMemberConnectionState;
|
||||
sharingScreen$: Behavior<boolean | undefined>;
|
||||
toggleScreenSharing: (() => void) | null;
|
||||
|
||||
// deprecated fields
|
||||
/** @deprecated use state instead*/
|
||||
homeserverConnected$: Behavior<boolean>;
|
||||
/** @deprecated use state instead*/
|
||||
connected$: Behavior<boolean>;
|
||||
// this needs to be discussed
|
||||
/** @deprecated use state instead*/
|
||||
reconnecting$: Behavior<boolean>;
|
||||
// also needs to be disccues
|
||||
/** @deprecated use state instead*/
|
||||
configError$: Behavior<ElementCallError | null>;
|
||||
} => {
|
||||
const state = {
|
||||
livekit$: new BehaviorSubject<LocalMemberLivekitState>({
|
||||
state: LivekitState.Uninitialized,
|
||||
}),
|
||||
matrix$: new BehaviorSubject<LocalMemberMatrixState>({
|
||||
state: MatrixState.Disconnected,
|
||||
}),
|
||||
};
|
||||
|
||||
// This should be used in a combineLatest with publisher$ to connect.
|
||||
// to make it possible to call startTracks before the preferredTransport$ has resolved.
|
||||
const shouldStartTracks$ = new BehaviorSubject(false);
|
||||
|
||||
// This should be used in a combineLatest with publisher$ to connect.
|
||||
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
|
||||
|
||||
// Drop Epoch data here since we will not combine this anymore
|
||||
const connection$ = scope.behavior(
|
||||
combineLatest(
|
||||
[connectionManager.connections$, localTransport$],
|
||||
(connections, transport) => {
|
||||
if (transport === undefined) return undefined;
|
||||
return connections.value.find((connection) =>
|
||||
areLivekitTransportsEqual(connection.transport, transport),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
/**
|
||||
* Whether we are connected to the MatrixRTC session.
|
||||
*/
|
||||
const homeserverConnected$ = scope.behavior(
|
||||
// To consider ourselves connected to MatrixRTC, we check the following:
|
||||
and$(
|
||||
// The client is connected to the sync loop
|
||||
(
|
||||
fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable<
|
||||
[SyncState]
|
||||
>
|
||||
).pipe(
|
||||
startWith([matrixRoom.client.getSyncState()]),
|
||||
map(([state]) => state === SyncState.Syncing),
|
||||
),
|
||||
// Room state observed by session says we're connected
|
||||
fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
|
||||
startWith(null),
|
||||
map(() => matrixRTCSession.membershipStatus === Status.Connected),
|
||||
),
|
||||
// Also watch out for warnings that we've likely hit a timeout and our
|
||||
// delayed leave event is being sent (this condition is here because it
|
||||
// provides an earlier warning than the sync loop timeout, and we wouldn't
|
||||
// see the actual leave event until we reconnect to the sync loop)
|
||||
fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe(
|
||||
startWith(null),
|
||||
map(() => matrixRTCSession.probablyLeft !== true),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// /**
|
||||
// * Whether we are "fully" connected to the call. Accounts for both the
|
||||
// * connection to the MatrixRTC session and the LiveKit publish connection.
|
||||
// */
|
||||
// // TODO use this in combination with the MemberState.
|
||||
const connected$ = scope.behavior(
|
||||
and$(
|
||||
homeserverConnected$,
|
||||
connection$.pipe(
|
||||
switchMap((c) =>
|
||||
c
|
||||
? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom"))
|
||||
: of(false),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const publisher$ = scope.behavior(
|
||||
connection$.pipe(
|
||||
map((connection) =>
|
||||
connection
|
||||
? new Publisher(
|
||||
scope,
|
||||
connection,
|
||||
mediaDevices,
|
||||
muteStates,
|
||||
e2eeLivekitOptions,
|
||||
trackProcessorState$,
|
||||
)
|
||||
: null,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
combineLatest(
|
||||
[publisher$, shouldStartTracks$],
|
||||
(publisher, shouldStartTracks) => {
|
||||
if (publisher && shouldStartTracks) {
|
||||
publisher
|
||||
.createAndSetupTracks()
|
||||
.then((tracks) => {
|
||||
tracks$.next(tracks);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error("Error creating tracks:", error);
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// MATRIX RELATED
|
||||
|
||||
// /**
|
||||
// * Whether we should tell the user that we're reconnecting to the call.
|
||||
// */
|
||||
// DISCUSSION is there a better way to do this?
|
||||
// sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
|
||||
const reconnecting$ = scope.behavior(
|
||||
connected$.pipe(
|
||||
// We are reconnecting if we previously had some successful initial
|
||||
// connection but are now disconnected
|
||||
scan(
|
||||
({ connectedPreviously }, connectedNow) => ({
|
||||
connectedPreviously: connectedPreviously || connectedNow,
|
||||
reconnecting: connectedPreviously && !connectedNow,
|
||||
}),
|
||||
{ connectedPreviously: false, reconnecting: false },
|
||||
),
|
||||
map(({ reconnecting }) => reconnecting),
|
||||
),
|
||||
);
|
||||
|
||||
const startTracks = (): Behavior<LocalTrack[]> => {
|
||||
shouldStartTracks$.next(true);
|
||||
return tracks$;
|
||||
};
|
||||
|
||||
const requestConnect = (): LocalMemberConnectionState => {
|
||||
if (state.livekit$.value === null) {
|
||||
startTracks();
|
||||
state.livekit$.next({ state: LivekitState.Connecting });
|
||||
combineLatest([publisher$, tracks$], (publisher, tracks) => {
|
||||
publisher
|
||||
?.startPublishing()
|
||||
.then(() => {
|
||||
state.livekit$.next({ state: LivekitState.Connected });
|
||||
})
|
||||
.catch((error) => {
|
||||
state.livekit$.next({ state: LivekitState.Error, error });
|
||||
});
|
||||
});
|
||||
}
|
||||
if (state.matrix$.value.state !== MatrixState.Disconnected) {
|
||||
state.matrix$.next({ state: MatrixState.Connecting });
|
||||
localTransport$.pipe(
|
||||
tap((transport) => {
|
||||
if (transport !== undefined) {
|
||||
enterRTCSession(matrixRTCSession, transport, options.value).catch(
|
||||
(error) => {
|
||||
logger.error(error);
|
||||
},
|
||||
);
|
||||
} else {
|
||||
logger.info("Waiting for transport to enter rtc session");
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
return state;
|
||||
};
|
||||
|
||||
const requestDisconnect = (): Behavior<LocalMemberLivekitState> | null => {
|
||||
if (state.livekit$.value.state !== LivekitState.Connected) return null;
|
||||
state.livekit$.next({ state: LivekitState.Disconnecting });
|
||||
combineLatest([publisher$, tracks$], (publisher, tracks) => {
|
||||
publisher
|
||||
?.stopPublishing()
|
||||
.then(() => {
|
||||
tracks.forEach((track) => track.stop());
|
||||
state.livekit$.next({ state: LivekitState.Disconnected });
|
||||
})
|
||||
.catch((error) => {
|
||||
state.livekit$.next({ state: LivekitState.Error, error });
|
||||
});
|
||||
});
|
||||
|
||||
return state.livekit$;
|
||||
};
|
||||
|
||||
// Pause upstream of all local media tracks when we're disconnected from
|
||||
// MatrixRTC, because it can be an unpleasant surprise for the app to say
|
||||
// 'reconnecting' and yet still be transmitting your media to others.
|
||||
// We use matrixConnected$ rather than reconnecting$ because we want to
|
||||
// pause tracks during the initial joining sequence too until we're sure
|
||||
// that our own media is displayed on screen.
|
||||
combineLatest([connection$, homeserverConnected$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([connection, connected]) => {
|
||||
if (connection?.state$.value.state !== "ConnectedToLkRoom") return;
|
||||
const publications =
|
||||
connection.livekitRoom.localParticipant.trackPublications.values();
|
||||
if (connected) {
|
||||
for (const p of publications) {
|
||||
if (p.track?.isUpstreamPaused === true) {
|
||||
const kind = p.track.kind;
|
||||
logger.log(`Resuming ${kind} track (MatrixRTC connection present)`);
|
||||
p.track
|
||||
.resumeUpstream()
|
||||
.catch((e) =>
|
||||
logger.error(
|
||||
`Failed to resume ${kind} track after MatrixRTC reconnection`,
|
||||
e,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const p of publications) {
|
||||
if (p.track?.isUpstreamPaused === false) {
|
||||
const kind = p.track.kind;
|
||||
logger.log(
|
||||
`Pausing ${kind} track (uncertain MatrixRTC connection)`,
|
||||
);
|
||||
p.track
|
||||
.pauseUpstream()
|
||||
.catch((e) =>
|
||||
logger.error(
|
||||
`Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
|
||||
e,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const configError$ = new BehaviorSubject<ElementCallError | null>(null);
|
||||
// TODO I do not fully understand what this does.
|
||||
// Is it needed?
|
||||
// Is this at the right place?
|
||||
// Can this be simplified?
|
||||
// Start and stop session membership as needed
|
||||
scope.reconcile(localTransport$, async (advertised) => {
|
||||
if (advertised !== null && advertised !== undefined) {
|
||||
try {
|
||||
configError$.next(null);
|
||||
await enterRTCSession(matrixRTCSession, advertised, options.value);
|
||||
} catch (e) {
|
||||
logger.error("Error entering RTC session", e);
|
||||
}
|
||||
|
||||
// Update our member event when our mute state changes.
|
||||
const intentScope = new ObservableScope();
|
||||
intentScope.reconcile(muteStates.video.enabled$, async (videoEnabled) =>
|
||||
matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
|
||||
);
|
||||
|
||||
return async (): Promise<void> => {
|
||||
intentScope.end();
|
||||
// Only sends Matrix leave event. The LiveKit session will disconnect
|
||||
// as soon as either the stopConnection$ handler above gets to it or
|
||||
// the view model is destroyed.
|
||||
try {
|
||||
await matrixRTCSession.leaveRoomSession();
|
||||
} catch (e) {
|
||||
logger.error("Error leaving RTC session", e);
|
||||
}
|
||||
try {
|
||||
await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
|
||||
} catch (e) {
|
||||
logger.error("Failed to send hangup action", e);
|
||||
}
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Returns undefined if scrennSharing is not yet ready.
|
||||
*/
|
||||
const sharingScreen$ = scope.behavior(
|
||||
connection$.pipe(
|
||||
switchMap((c) => {
|
||||
if (!c) return of(undefined);
|
||||
if (c.state$.value.state === "ConnectedToLkRoom")
|
||||
return observeSharingScreen$(c.livekitRoom.localParticipant);
|
||||
return of(false);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const toggleScreenSharing =
|
||||
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
|
||||
!getUrlParams().hideScreensharing
|
||||
? (): void =>
|
||||
// If a connection is ready...
|
||||
void connection$
|
||||
.pipe(
|
||||
// I dont see why we need this. isnt the check later on superseeding it?
|
||||
takeWhile(
|
||||
(c) =>
|
||||
c !== undefined && c.state$.value.state !== "FailedToStart",
|
||||
),
|
||||
switchMap((c) =>
|
||||
c?.state$.value.state === "ConnectedToLkRoom" ? of(c) : NEVER,
|
||||
),
|
||||
take(1),
|
||||
scope.bind(),
|
||||
)
|
||||
// ...toggle screen sharing.
|
||||
.subscribe(
|
||||
(c) =>
|
||||
void c.livekitRoom.localParticipant
|
||||
.setScreenShareEnabled(!sharingScreen$.value, {
|
||||
audio: true,
|
||||
selfBrowserSurface: "include",
|
||||
surfaceSwitching: "include",
|
||||
systemAudio: "include",
|
||||
})
|
||||
.catch(logger.error),
|
||||
)
|
||||
: null;
|
||||
|
||||
// we do not need all the auto waiting since we can just check via sharingScreen$.value !== undefined
|
||||
let alternativeScreenshareToggle: (() => void) | null = null;
|
||||
if (
|
||||
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
|
||||
!getUrlParams().hideScreensharing
|
||||
) {
|
||||
alternativeScreenshareToggle = (): void =>
|
||||
void connection$.value?.livekitRoom.localParticipant
|
||||
.setScreenShareEnabled(!sharingScreen$.value, {
|
||||
audio: true,
|
||||
selfBrowserSurface: "include",
|
||||
surfaceSwitching: "include",
|
||||
systemAudio: "include",
|
||||
})
|
||||
.catch(logger.error);
|
||||
}
|
||||
logger.log(
|
||||
"alternativeScreenshareToggle so that it is used",
|
||||
alternativeScreenshareToggle,
|
||||
);
|
||||
|
||||
return {
|
||||
startTracks,
|
||||
requestConnect,
|
||||
requestDisconnect,
|
||||
connectionState: state,
|
||||
homeserverConnected$,
|
||||
connected$,
|
||||
reconnecting$,
|
||||
configError$,
|
||||
sharingScreen$,
|
||||
toggleScreenSharing,
|
||||
};
|
||||
};
|
||||
169
src/state/CallViewModel/localMember/LocalTransport.ts
Normal file
169
src/state/CallViewModel/localMember/LocalTransport.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CallMembership,
|
||||
isLivekitTransport,
|
||||
type LivekitTransportConfig,
|
||||
type LivekitTransport,
|
||||
isLivekitTransportConfig,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { combineLatest, distinctUntilChanged, first, from } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
|
||||
import { type Behavior } from "../../Behavior.ts";
|
||||
import {
|
||||
type Epoch,
|
||||
mapEpoch,
|
||||
type ObservableScope,
|
||||
} from "../../ObservableScope.ts";
|
||||
import { Config } from "../../../config/Config.ts";
|
||||
import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
|
||||
import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts";
|
||||
|
||||
/*
|
||||
* - get well known
|
||||
* - get oldest membership
|
||||
* - get transport to use
|
||||
* - get openId + jwt token
|
||||
* - wait for createTrack() call
|
||||
* - create tracks
|
||||
* - wait for join() call
|
||||
* - Publisher.publishTracks()
|
||||
* - send join state/sticky event
|
||||
*/
|
||||
interface Props {
|
||||
scope: ObservableScope;
|
||||
memberships$: Behavior<Epoch<CallMembership[]>>;
|
||||
client: MatrixClient;
|
||||
roomId: string;
|
||||
useOldestMember$: Behavior<boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
* This class is responsible for managing the local transport.
|
||||
* "Which transport is the local member going to use"
|
||||
*
|
||||
* @prop useOldestMember Whether to use the same transport as the oldest member.
|
||||
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
|
||||
*/
|
||||
export const createLocalTransport$ = ({
|
||||
scope,
|
||||
memberships$,
|
||||
client,
|
||||
roomId,
|
||||
useOldestMember$,
|
||||
}: Props): Behavior<LivekitTransport | undefined> => {
|
||||
/**
|
||||
* The transport over which we should be actively publishing our media.
|
||||
* undefined when not joined.
|
||||
*/
|
||||
const oldestMemberTransport$ = scope.behavior(
|
||||
memberships$.pipe(
|
||||
mapEpoch((memberships) => memberships[0].getTransport(memberships[0])),
|
||||
first((t) => t != undefined && isLivekitTransport(t)),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
|
||||
/**
|
||||
* The transport that we would personally prefer to publish on (if not for the
|
||||
* transport preferences of others, perhaps).
|
||||
*/
|
||||
const preferredTransport$: Behavior<LivekitTransport | undefined> =
|
||||
scope.behavior(from(makeTransport(client, roomId)), undefined);
|
||||
|
||||
/**
|
||||
* The transport we should advertise in our MatrixRTC membership.
|
||||
*/
|
||||
const advertisedTransport$ = scope.behavior(
|
||||
combineLatest(
|
||||
[useOldestMember$, oldestMemberTransport$, preferredTransport$],
|
||||
(useOldestMember, oldestMemberTransport, preferredTransport) =>
|
||||
useOldestMember ? oldestMemberTransport : preferredTransport,
|
||||
).pipe<LivekitTransport>(distinctUntilChanged(deepCompare)),
|
||||
undefined,
|
||||
);
|
||||
return advertisedTransport$;
|
||||
};
|
||||
|
||||
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
|
||||
|
||||
async function makeTransportInternal(
|
||||
client: MatrixClient,
|
||||
roomId: string,
|
||||
): Promise<LivekitTransport> {
|
||||
logger.log("Searching for a preferred transport");
|
||||
//TODO refactor this to use the jwt service returned alias.
|
||||
const livekitAlias = roomId;
|
||||
|
||||
// TODO-MULTI-SFU: Either remove this dev tool or make it more official
|
||||
const urlFromStorage =
|
||||
localStorage.getItem("robin-matrixrtc-auth") ??
|
||||
localStorage.getItem("timo-focus-url");
|
||||
if (urlFromStorage !== null) {
|
||||
const transportFromStorage: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: urlFromStorage,
|
||||
livekit_alias: livekitAlias,
|
||||
};
|
||||
logger.log(
|
||||
"Using LiveKit transport from local storage: ",
|
||||
transportFromStorage,
|
||||
);
|
||||
return transportFromStorage;
|
||||
}
|
||||
|
||||
// Prioritize the .well-known/matrix/client, if available, over the configured SFU
|
||||
const domain = client.getDomain();
|
||||
if (domain) {
|
||||
// we use AutoDiscovery instead of relying on the MatrixClient having already
|
||||
// been fully configured and started
|
||||
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
|
||||
FOCI_WK_KEY
|
||||
];
|
||||
if (Array.isArray(wellKnownFoci)) {
|
||||
const transport: LivekitTransportConfig | undefined = wellKnownFoci.find(
|
||||
(f) => f && isLivekitTransportConfig(f),
|
||||
);
|
||||
if (transport !== undefined) {
|
||||
logger.log("Using LiveKit transport from .well-known: ", transport);
|
||||
return { ...transport, livekit_alias: livekitAlias };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const urlFromConf = Config.get().livekit?.livekit_service_url;
|
||||
if (urlFromConf) {
|
||||
const transportFromConf: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: urlFromConf,
|
||||
livekit_alias: livekitAlias,
|
||||
};
|
||||
logger.log("Using LiveKit transport from config: ", transportFromConf);
|
||||
return transportFromConf;
|
||||
}
|
||||
|
||||
throw new MatrixRTCTransportMissingError(domain ?? "");
|
||||
}
|
||||
|
||||
async function makeTransport(
|
||||
client: MatrixClient,
|
||||
roomId: string,
|
||||
): Promise<LivekitTransport> {
|
||||
const transport = await makeTransportInternal(client, roomId);
|
||||
// this will call the jwt/sfu/get endpoint to pre create the livekit room.
|
||||
await getSFUConfigWithOpenID(
|
||||
client,
|
||||
transport.livekit_service_url,
|
||||
transport.livekit_alias,
|
||||
);
|
||||
return transport;
|
||||
}
|
||||
313
src/state/CallViewModel/localMember/Publisher.ts
Normal file
313
src/state/CallViewModel/localMember/Publisher.ts
Normal file
@@ -0,0 +1,313 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
import {
|
||||
type E2EEOptions,
|
||||
LocalVideoTrack,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
ConnectionState as LivekitConnectionState,
|
||||
} from "livekit-client";
|
||||
import {
|
||||
map,
|
||||
NEVER,
|
||||
type Observable,
|
||||
type Subscription,
|
||||
switchMap,
|
||||
} from "rxjs";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import type { Behavior } from "../../Behavior.ts";
|
||||
import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts";
|
||||
import type { MuteStates } from "../../MuteStates.ts";
|
||||
import {
|
||||
type ProcessorState,
|
||||
trackProcessorSync,
|
||||
} from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import { getUrlParams } from "../../../UrlParams.ts";
|
||||
import { observeTrackReference$ } from "../../MediaViewModel.ts";
|
||||
import { type Connection } from "../CallViewModel/remoteMembers/Connection.ts";
|
||||
import { type ObservableScope } from "../../ObservableScope.ts";
|
||||
|
||||
/**
|
||||
* A wrapper for a Connection object.
|
||||
* This wrapper will manage the connection used to publish to the LiveKit room.
|
||||
* The Publisher is also responsible for creating the media tracks.
|
||||
*/
|
||||
export class Publisher {
|
||||
public tracks: LocalTrack<Track.Kind>[] = [];
|
||||
/**
|
||||
* Creates a new Publisher.
|
||||
* @param scope - The observable scope to use for managing the publisher.
|
||||
* @param connection - The connection to use for publishing.
|
||||
* @param devices - The media devices to use for audio and video input.
|
||||
* @param muteStates - The mute states for audio and video.
|
||||
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
|
||||
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
|
||||
*/
|
||||
public constructor(
|
||||
private scope: ObservableScope,
|
||||
private connection: Connection,
|
||||
devices: MediaDevices,
|
||||
private readonly muteStates: MuteStates,
|
||||
e2eeLivekitOptions: E2EEOptions | undefined,
|
||||
trackerProcessorState$: Behavior<ProcessorState>,
|
||||
private logger?: Logger,
|
||||
) {
|
||||
this.logger?.info("[PublishConnection] Create LiveKit room");
|
||||
const { controlledAudioDevices } = getUrlParams();
|
||||
|
||||
const room = connection.livekitRoom;
|
||||
|
||||
room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => {
|
||||
this.logger?.error("Failed to set E2EE enabled on room", e);
|
||||
});
|
||||
|
||||
// Setup track processor syncing (blur)
|
||||
this.observeTrackProcessors(scope, room, trackerProcessorState$);
|
||||
// Observe media device changes and update LiveKit active devices accordingly
|
||||
this.observeMediaDevices(scope, devices, controlledAudioDevices);
|
||||
|
||||
this.workaroundRestartAudioInputTrackChrome(devices, scope);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the connection to LiveKit and publish local tracks.
|
||||
*
|
||||
* This will:
|
||||
* wait for the connection to be ready.
|
||||
// * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
|
||||
// * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
|
||||
// * 3. Connect to the configured LiveKit room.
|
||||
// * 4. Create local audio and video tracks based on the current mute states and publish them to the room.
|
||||
*
|
||||
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
|
||||
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
|
||||
*/
|
||||
public async createAndSetupTracks(): Promise<LocalTrack[]> {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
// Observe mute state changes and update LiveKit microphone/camera states accordingly
|
||||
this.observeMuteStates(this.scope);
|
||||
|
||||
// TODO: This should be an autostarted connection no need to start here. just check the connection state.
|
||||
// TODO: This will fetch the JWT token. Perhaps we could keep it preloaded
|
||||
// instead? This optimization would only be safe for a publish connection,
|
||||
// because we don't want to leak the user's intent to perhaps join a call to
|
||||
// remote servers before they actually commit to it.
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const sub = this.connection.state$.subscribe((s) => {
|
||||
if (s.state !== "FailedToStart") {
|
||||
reject(new Error("Disconnected from LiveKit server"));
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
} finally {
|
||||
sub.unsubscribe();
|
||||
}
|
||||
// TODO-MULTI-SFU: Prepublish a microphone track
|
||||
const audio = this.muteStates.audio.enabled$.value;
|
||||
const video = this.muteStates.video.enabled$.value;
|
||||
// createTracks throws if called with audio=false and video=false
|
||||
if (audio || video) {
|
||||
// TODO this can still throw errors? It will also prompt for permissions if not already granted
|
||||
this.tracks = await lkRoom.localParticipant.createTracks({
|
||||
audio,
|
||||
video,
|
||||
});
|
||||
}
|
||||
return this.tracks;
|
||||
}
|
||||
|
||||
public async startPublishing(): Promise<LocalTrack[]> {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const sub = this.connection.state$.subscribe((s) => {
|
||||
switch (s.state) {
|
||||
case "ConnectedToLkRoom":
|
||||
resolve();
|
||||
break;
|
||||
case "FailedToStart":
|
||||
reject(new Error("Failed to connect to LiveKit server"));
|
||||
break;
|
||||
default:
|
||||
this.logger?.info("waiting for connection: ", s.state);
|
||||
}
|
||||
});
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
} finally {
|
||||
sub.unsubscribe();
|
||||
}
|
||||
for (const track of this.tracks) {
|
||||
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
|
||||
// with a timeout.
|
||||
await lkRoom.localParticipant.publishTrack(track);
|
||||
|
||||
// TODO: check if the connection is still active? and break the loop if not?
|
||||
}
|
||||
return this.tracks;
|
||||
}
|
||||
|
||||
public async stopPublishing(): Promise<void> {
|
||||
// TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
|
||||
// actually has the right lifetime
|
||||
this.muteStates.audio.unsetHandler();
|
||||
this.muteStates.video.unsetHandler();
|
||||
|
||||
const localParticipant = this.connection.livekitRoom.localParticipant;
|
||||
const tracks: LocalTrack[] = [];
|
||||
const addToTracksIfDefined = (p: LocalTrackPublication): void => {
|
||||
if (p.track !== undefined) tracks.push(p.track);
|
||||
};
|
||||
localParticipant.trackPublications.forEach(addToTracksIfDefined);
|
||||
await localParticipant.unpublishTracks(tracks);
|
||||
}
|
||||
|
||||
/// Private methods
|
||||
|
||||
// Restart the audio input track whenever we detect that the active media
|
||||
// device has changed to refer to a different hardware device. We do this
|
||||
// for the sake of Chrome, which provides a "default" device that is meant
|
||||
// to match the system's default audio input, whatever that may be.
|
||||
// This is special-cased for only audio inputs because we need to dig around
|
||||
// in the LocalParticipant object for the track object and there's not a nice
|
||||
// way to do that generically. There is usually no OS-level default video capture
|
||||
// device anyway, and audio outputs work differently.
|
||||
private workaroundRestartAudioInputTrackChrome(
|
||||
devices: MediaDevices,
|
||||
scope: ObservableScope,
|
||||
): void {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
devices.audioInput.selected$
|
||||
.pipe(
|
||||
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
|
||||
scope.bind(),
|
||||
)
|
||||
.subscribe(() => {
|
||||
if (lkRoom.state != LivekitConnectionState.Connected) return;
|
||||
const activeMicTrack = Array.from(
|
||||
lkRoom.localParticipant.audioTrackPublications.values(),
|
||||
).find((d) => d.source === Track.Source.Microphone)?.track;
|
||||
|
||||
if (
|
||||
activeMicTrack &&
|
||||
// only restart if the stream is still running: LiveKit will detect
|
||||
// when a track stops & restart appropriately, so this is not our job.
|
||||
// Plus, we need to avoid restarting again if the track is already in
|
||||
// the process of being restarted.
|
||||
activeMicTrack.mediaStreamTrack.readyState !== "ended"
|
||||
) {
|
||||
// Restart the track, which will cause Livekit to do another
|
||||
// getUserMedia() call with deviceId: default to get the *new* default device.
|
||||
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
|
||||
// the deviceId hasn't changed (was & still is default).
|
||||
lkRoom.localParticipant
|
||||
.getTrackPublication(Track.Source.Microphone)
|
||||
?.audioTrack?.restartTrack()
|
||||
.catch((e) => {
|
||||
this.logger?.error(`Failed to restart audio device track`, e);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Observe changes in the selected media devices and update the LiveKit room accordingly.
|
||||
private observeMediaDevices(
|
||||
scope: ObservableScope,
|
||||
devices: MediaDevices,
|
||||
controlledAudioDevices: boolean,
|
||||
): void {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
const syncDevice = (
|
||||
kind: MediaDeviceKind,
|
||||
selected$: Observable<SelectedDevice | undefined>,
|
||||
): Subscription =>
|
||||
selected$.pipe(scope.bind()).subscribe((device) => {
|
||||
if (lkRoom.state != LivekitConnectionState.Connected) return;
|
||||
// if (this.connectionState$.value !== ConnectionState.Connected) return;
|
||||
this.logger?.info(
|
||||
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
|
||||
lkRoom.getActiveDevice(kind),
|
||||
" !== ",
|
||||
device?.id,
|
||||
);
|
||||
if (
|
||||
device !== undefined &&
|
||||
lkRoom.getActiveDevice(kind) !== device.id
|
||||
) {
|
||||
lkRoom
|
||||
.switchActiveDevice(kind, device.id)
|
||||
.catch((e) =>
|
||||
this.logger?.error(
|
||||
`Failed to sync ${kind} device with LiveKit`,
|
||||
e,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
syncDevice("audioinput", devices.audioInput.selected$);
|
||||
if (!controlledAudioDevices)
|
||||
syncDevice("audiooutput", devices.audioOutput.selected$);
|
||||
syncDevice("videoinput", devices.videoInput.selected$);
|
||||
}
|
||||
|
||||
/**
|
||||
* Observe changes in the mute states and update the LiveKit room accordingly.
|
||||
* @param scope
|
||||
* @private
|
||||
*/
|
||||
private observeMuteStates(scope: ObservableScope): void {
|
||||
const lkRoom = this.connection.livekitRoom;
|
||||
this.muteStates.audio.setHandler(async (desired) => {
|
||||
try {
|
||||
await lkRoom.localParticipant.setMicrophoneEnabled(desired);
|
||||
} catch (e) {
|
||||
this.logger?.error(
|
||||
"Failed to update LiveKit audio input mute state",
|
||||
e,
|
||||
);
|
||||
}
|
||||
return lkRoom.localParticipant.isMicrophoneEnabled;
|
||||
});
|
||||
this.muteStates.video.setHandler(async (desired) => {
|
||||
try {
|
||||
await lkRoom.localParticipant.setCameraEnabled(desired);
|
||||
} catch (e) {
|
||||
this.logger?.error(
|
||||
"Failed to update LiveKit video input mute state",
|
||||
e,
|
||||
);
|
||||
}
|
||||
return lkRoom.localParticipant.isCameraEnabled;
|
||||
});
|
||||
}
|
||||
|
||||
private observeTrackProcessors(
|
||||
scope: ObservableScope,
|
||||
room: LivekitRoom,
|
||||
trackerProcessorState$: Behavior<ProcessorState>,
|
||||
): void {
|
||||
const track$ = scope.behavior(
|
||||
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
|
||||
map((trackRef) => {
|
||||
const track = trackRef?.publication?.track;
|
||||
return track instanceof LocalVideoTrack ? track : null;
|
||||
}),
|
||||
),
|
||||
);
|
||||
trackProcessorSync(track$, trackerProcessorState$);
|
||||
}
|
||||
}
|
||||
700
src/state/CallViewModel/remoteMembers/Connection.test.ts
Normal file
700
src/state/CallViewModel/remoteMembers/Connection.test.ts
Normal file
@@ -0,0 +1,700 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
afterEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
type Mock,
|
||||
type MockedObject,
|
||||
onTestFinished,
|
||||
vi,
|
||||
} from "vitest";
|
||||
import { BehaviorSubject, of } from "rxjs";
|
||||
import {
|
||||
type LocalParticipant,
|
||||
type RemoteParticipant,
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
type RoomOptions,
|
||||
} from "livekit-client";
|
||||
import fetchMock from "fetch-mock";
|
||||
import EventEmitter from "events";
|
||||
import { type IOpenIDToken } from "matrix-js-sdk";
|
||||
|
||||
import type {
|
||||
CallMembership,
|
||||
LivekitTransport,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
type ConnectionOpts,
|
||||
type ConnectionState,
|
||||
type PublishingParticipant,
|
||||
} from "./Connection.ts";
|
||||
import { ObservableScope } from "../../ObservableScope.ts";
|
||||
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
|
||||
import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
|
||||
import { mockMediaDevices, mockMuteStates } from "../../../utils/test.ts";
|
||||
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import { type MuteStates } from "../../MuteStates.ts";
|
||||
|
||||
let testScope: ObservableScope;
|
||||
|
||||
let client: MockedObject<OpenIDClientParts>;
|
||||
|
||||
let fakeLivekitRoom: MockedObject<LivekitRoom>;
|
||||
|
||||
let localParticipantEventEmiter: EventEmitter;
|
||||
let fakeLocalParticipant: MockedObject<LocalParticipant>;
|
||||
|
||||
let fakeRoomEventEmiter: EventEmitter;
|
||||
let fakeMembershipsFocusMap$: BehaviorSubject<
|
||||
{ membership: CallMembership; transport: LivekitTransport }[]
|
||||
>;
|
||||
|
||||
const livekitFocus: LivekitTransport = {
|
||||
livekit_alias: "!roomID:example.org",
|
||||
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
|
||||
type: "livekit",
|
||||
};
|
||||
|
||||
function setupTest(): void {
|
||||
testScope = new ObservableScope();
|
||||
client = vi.mocked<OpenIDClientParts>({
|
||||
getOpenIdToken: vi.fn().mockResolvedValue({
|
||||
access_token: "rYsmGUEwNjKgJYyeNUkZseJN",
|
||||
token_type: "Bearer",
|
||||
matrix_server_name: "example.org",
|
||||
expires_in: 3600,
|
||||
}),
|
||||
getDeviceId: vi.fn().mockReturnValue("ABCDEF"),
|
||||
} as unknown as OpenIDClientParts);
|
||||
fakeMembershipsFocusMap$ = new BehaviorSubject<
|
||||
{ membership: CallMembership; transport: LivekitTransport }[]
|
||||
>([]);
|
||||
|
||||
localParticipantEventEmiter = new EventEmitter();
|
||||
|
||||
fakeLocalParticipant = vi.mocked<LocalParticipant>({
|
||||
identity: "@me:example.org",
|
||||
isMicrophoneEnabled: vi.fn().mockReturnValue(true),
|
||||
getTrackPublication: vi.fn().mockReturnValue(undefined),
|
||||
on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter),
|
||||
off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter),
|
||||
addListener: localParticipantEventEmiter.addListener.bind(
|
||||
localParticipantEventEmiter,
|
||||
),
|
||||
removeListener: localParticipantEventEmiter.removeListener.bind(
|
||||
localParticipantEventEmiter,
|
||||
),
|
||||
removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(
|
||||
localParticipantEventEmiter,
|
||||
),
|
||||
} as unknown as LocalParticipant);
|
||||
fakeRoomEventEmiter = new EventEmitter();
|
||||
|
||||
fakeLivekitRoom = vi.mocked<LivekitRoom>({
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
remoteParticipants: new Map(),
|
||||
localParticipant: fakeLocalParticipant,
|
||||
state: ConnectionState.Disconnected,
|
||||
on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
|
||||
off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
|
||||
addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
|
||||
removeListener:
|
||||
fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
|
||||
removeAllListeners:
|
||||
fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
|
||||
setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as LivekitRoom);
|
||||
}
|
||||
|
||||
function setupRemoteConnection(): RemoteConnection {
|
||||
const opts: ConnectionOpts = {
|
||||
client: client,
|
||||
transport: livekitFocus,
|
||||
remoteTransports$: fakeMembershipsFocusMap$,
|
||||
scope: testScope,
|
||||
livekitRoomFactory: () => fakeLivekitRoom,
|
||||
};
|
||||
|
||||
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
|
||||
jwt: "ATOKEN",
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
fakeLivekitRoom.connect.mockResolvedValue(undefined);
|
||||
|
||||
return new RemoteConnection(opts, undefined);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
fetchMock.reset();
|
||||
});
|
||||
|
||||
describe("Start connection states", () => {
|
||||
it("start in initialized state", () => {
|
||||
setupTest();
|
||||
|
||||
const opts: ConnectionOpts = {
|
||||
client: client,
|
||||
transport: livekitFocus,
|
||||
remoteTransports$: fakeMembershipsFocusMap$,
|
||||
scope: testScope,
|
||||
livekitRoomFactory: () => fakeLivekitRoom,
|
||||
};
|
||||
const connection = new RemoteConnection(opts, undefined);
|
||||
|
||||
expect(connection.state$.getValue().state).toEqual("Initialized");
|
||||
});
|
||||
|
||||
it("fail to getOpenId token then error state", async () => {
|
||||
setupTest();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const opts: ConnectionOpts = {
|
||||
client: client,
|
||||
transport: livekitFocus,
|
||||
remoteTransports$: fakeMembershipsFocusMap$,
|
||||
scope: testScope,
|
||||
livekitRoomFactory: () => fakeLivekitRoom,
|
||||
};
|
||||
|
||||
const connection = new RemoteConnection(opts, undefined);
|
||||
|
||||
const capturedStates: ConnectionState[] = [];
|
||||
const s = connection.state$.subscribe((value) => {
|
||||
capturedStates.push(value);
|
||||
});
|
||||
onTestFinished(() => s.unsubscribe());
|
||||
|
||||
const deferred = Promise.withResolvers<IOpenIDToken>();
|
||||
|
||||
client.getOpenIdToken.mockImplementation(
|
||||
async (): Promise<IOpenIDToken> => {
|
||||
return await deferred.promise;
|
||||
},
|
||||
);
|
||||
|
||||
connection.start().catch(() => {
|
||||
// expected to throw
|
||||
});
|
||||
|
||||
let capturedState = capturedStates.pop();
|
||||
expect(capturedState).toBeDefined();
|
||||
expect(capturedState!.state).toEqual("FetchingConfig");
|
||||
|
||||
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
capturedState = capturedStates.pop();
|
||||
if (capturedState!.state === "FailedToStart") {
|
||||
expect(capturedState!.error.message).toEqual("Something went wrong");
|
||||
expect(capturedState!.transport.livekit_alias).toEqual(
|
||||
livekitFocus.livekit_alias,
|
||||
);
|
||||
} else {
|
||||
expect.fail(
|
||||
"Expected FailedToStart state but got " + capturedState?.state,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("fail to get JWT token and error state", async () => {
|
||||
setupTest();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const opts: ConnectionOpts = {
|
||||
client: client,
|
||||
transport: livekitFocus,
|
||||
remoteTransports$: fakeMembershipsFocusMap$,
|
||||
scope: testScope,
|
||||
livekitRoomFactory: () => fakeLivekitRoom,
|
||||
};
|
||||
|
||||
const connection = new RemoteConnection(opts, undefined);
|
||||
|
||||
const capturedStates: ConnectionState[] = [];
|
||||
const s = connection.state$.subscribe((value) => {
|
||||
capturedStates.push(value);
|
||||
});
|
||||
onTestFinished(() => s.unsubscribe());
|
||||
|
||||
const deferredSFU = Promise.withResolvers<void>();
|
||||
// mock the /sfu/get call
|
||||
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => {
|
||||
await deferredSFU.promise;
|
||||
return {
|
||||
status: 500,
|
||||
body: "Internal Server Error",
|
||||
};
|
||||
});
|
||||
|
||||
connection.start().catch(() => {
|
||||
// expected to throw
|
||||
});
|
||||
|
||||
let capturedState = capturedStates.pop();
|
||||
expect(capturedState).toBeDefined();
|
||||
expect(capturedState?.state).toEqual("FetchingConfig");
|
||||
|
||||
deferredSFU.resolve();
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
capturedState = capturedStates.pop();
|
||||
|
||||
if (capturedState?.state === "FailedToStart") {
|
||||
expect(capturedState?.error.message).toContain(
|
||||
"SFU Config fetch failed with exception Error",
|
||||
);
|
||||
expect(capturedState?.transport.livekit_alias).toEqual(
|
||||
livekitFocus.livekit_alias,
|
||||
);
|
||||
} else {
|
||||
expect.fail(
|
||||
"Expected FailedToStart state but got " + capturedState?.state,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("fail to connect to livekit error state", async () => {
|
||||
setupTest();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const opts: ConnectionOpts = {
|
||||
client: client,
|
||||
transport: livekitFocus,
|
||||
remoteTransports$: fakeMembershipsFocusMap$,
|
||||
scope: testScope,
|
||||
livekitRoomFactory: () => fakeLivekitRoom,
|
||||
};
|
||||
|
||||
const connection = new RemoteConnection(opts, undefined);
|
||||
|
||||
const capturedStates: ConnectionState[] = [];
|
||||
const s = connection.state$.subscribe((value) => {
|
||||
capturedStates.push(value);
|
||||
});
|
||||
onTestFinished(() => s.unsubscribe());
|
||||
|
||||
const deferredSFU = Promise.withResolvers<void>();
|
||||
// mock the /sfu/get call
|
||||
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
|
||||
jwt: "ATOKEN",
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
fakeLivekitRoom.connect.mockImplementation(async () => {
|
||||
await deferredSFU.promise;
|
||||
throw new Error("Failed to connect to livekit");
|
||||
});
|
||||
|
||||
connection.start().catch(() => {
|
||||
// expected to throw
|
||||
});
|
||||
|
||||
let capturedState = capturedStates.pop();
|
||||
expect(capturedState).toBeDefined();
|
||||
|
||||
expect(capturedState?.state).toEqual("FetchingConfig");
|
||||
|
||||
deferredSFU.resolve();
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
capturedState = capturedStates.pop();
|
||||
|
||||
if (capturedState && capturedState?.state === "FailedToStart") {
|
||||
expect(capturedState.error.message).toContain(
|
||||
"Failed to connect to livekit",
|
||||
);
|
||||
expect(capturedState.transport.livekit_alias).toEqual(
|
||||
livekitFocus.livekit_alias,
|
||||
);
|
||||
} else {
|
||||
expect.fail(
|
||||
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("connection states happy path", async () => {
|
||||
vi.useFakeTimers();
|
||||
setupTest();
|
||||
|
||||
const connection = setupRemoteConnection();
|
||||
|
||||
const capturedStates: ConnectionState[] = [];
|
||||
const s = connection.state$.subscribe((value) => {
|
||||
capturedStates.push(value);
|
||||
});
|
||||
onTestFinished(() => s.unsubscribe());
|
||||
|
||||
await connection.start();
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
const initialState = capturedStates.shift();
|
||||
expect(initialState?.state).toEqual("Initialized");
|
||||
const fetchingState = capturedStates.shift();
|
||||
expect(fetchingState?.state).toEqual("FetchingConfig");
|
||||
const connectingState = capturedStates.shift();
|
||||
expect(connectingState?.state).toEqual("ConnectingToLkRoom");
|
||||
const connectedState = capturedStates.shift();
|
||||
expect(connectedState?.state).toEqual("ConnectedToLkRoom");
|
||||
});
|
||||
|
||||
it("shutting down the scope should stop the connection", async () => {
|
||||
setupTest();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const connection = setupRemoteConnection();
|
||||
await connection.start();
|
||||
|
||||
const stopSpy = vi.spyOn(connection, "stop");
|
||||
testScope.end();
|
||||
|
||||
expect(stopSpy).toHaveBeenCalled();
|
||||
expect(fakeLivekitRoom.disconnect).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
function fakeRemoteLivekitParticipant(id: string): RemoteParticipant {
|
||||
return {
|
||||
identity: id,
|
||||
} as unknown as RemoteParticipant;
|
||||
}
|
||||
|
||||
function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership {
|
||||
return {
|
||||
userId,
|
||||
deviceId,
|
||||
} as unknown as CallMembership;
|
||||
}
|
||||
|
||||
describe("Publishing participants observations", () => {
|
||||
it("should emit the list of publishing participants", async () => {
|
||||
setupTest();
|
||||
|
||||
const connection = setupRemoteConnection();
|
||||
|
||||
const bobIsAPublisher = Promise.withResolvers<void>();
|
||||
const danIsAPublisher = Promise.withResolvers<void>();
|
||||
const observedPublishers: PublishingParticipant[][] = [];
|
||||
const s = connection.allLivekitParticipants$.subscribe((publishers) => {
|
||||
observedPublishers.push(publishers);
|
||||
if (
|
||||
publishers.some(
|
||||
(p) => p.participant?.identity === "@bob:example.org:DEV111",
|
||||
)
|
||||
) {
|
||||
bobIsAPublisher.resolve();
|
||||
}
|
||||
if (
|
||||
publishers.some(
|
||||
(p) => p.participant?.identity === "@dan:example.org:DEV333",
|
||||
)
|
||||
) {
|
||||
danIsAPublisher.resolve();
|
||||
}
|
||||
});
|
||||
onTestFinished(() => s.unsubscribe());
|
||||
// The publishingParticipants$ observable is derived from the current members of the
|
||||
// livekitRoom and the rtc membership in order to publish the members that are publishing
|
||||
// on this connection.
|
||||
|
||||
let participants: RemoteParticipant[] = [
|
||||
fakeRemoteLivekitParticipant("@alice:example.org:DEV000"),
|
||||
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
|
||||
fakeRemoteLivekitParticipant("@carol:example.org:DEV222"),
|
||||
fakeRemoteLivekitParticipant("@dan:example.org:DEV333"),
|
||||
];
|
||||
|
||||
// Let's simulate 3 members on the livekitRoom
|
||||
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
|
||||
new Map(participants.map((p) => [p.identity, p])),
|
||||
);
|
||||
|
||||
for (const participant of participants) {
|
||||
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
|
||||
}
|
||||
|
||||
// At this point there should be no publishers
|
||||
expect(observedPublishers.pop()!.length).toEqual(0);
|
||||
|
||||
const otherFocus: LivekitTransport = {
|
||||
livekit_alias: "!roomID:example.org",
|
||||
livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt",
|
||||
type: "livekit",
|
||||
};
|
||||
|
||||
const rtcMemberships = [
|
||||
// Say bob is on the same focus
|
||||
{
|
||||
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
|
||||
transport: livekitFocus,
|
||||
},
|
||||
// Alice and carol is on a different focus
|
||||
{
|
||||
membership: fakeRtcMemberShip("@alice:example.org", "DEV000"),
|
||||
transport: otherFocus,
|
||||
},
|
||||
{
|
||||
membership: fakeRtcMemberShip("@carol:example.org", "DEV222"),
|
||||
transport: otherFocus,
|
||||
},
|
||||
// NO DAVE YET
|
||||
];
|
||||
// signal this change in rtc memberships
|
||||
fakeMembershipsFocusMap$.next(rtcMemberships);
|
||||
|
||||
// We should have bob has a publisher now
|
||||
await bobIsAPublisher.promise;
|
||||
const publishers = observedPublishers.pop();
|
||||
expect(publishers?.length).toEqual(1);
|
||||
expect(publishers?.[0].participant?.identity).toEqual(
|
||||
"@bob:example.org:DEV111",
|
||||
);
|
||||
|
||||
// Now let's make dan join the rtc memberships
|
||||
rtcMemberships.push({
|
||||
membership: fakeRtcMemberShip("@dan:example.org", "DEV333"),
|
||||
transport: livekitFocus,
|
||||
});
|
||||
fakeMembershipsFocusMap$.next(rtcMemberships);
|
||||
|
||||
// We should have bob and dan has publishers now
|
||||
await danIsAPublisher.promise;
|
||||
const twoPublishers = observedPublishers.pop();
|
||||
expect(twoPublishers?.length).toEqual(2);
|
||||
expect(
|
||||
twoPublishers?.some(
|
||||
(p) => p.participant?.identity === "@bob:example.org:DEV111",
|
||||
),
|
||||
).toBeTruthy();
|
||||
expect(
|
||||
twoPublishers?.some(
|
||||
(p) => p.participant?.identity === "@dan:example.org:DEV333",
|
||||
),
|
||||
).toBeTruthy();
|
||||
|
||||
// Now let's make bob leave the livekit room
|
||||
participants = participants.filter(
|
||||
(p) => p.identity !== "@bob:example.org:DEV111",
|
||||
);
|
||||
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
|
||||
new Map(participants.map((p) => [p.identity, p])),
|
||||
);
|
||||
fakeRoomEventEmiter.emit(
|
||||
RoomEvent.ParticipantDisconnected,
|
||||
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
|
||||
);
|
||||
|
||||
const updatedPublishers = observedPublishers.pop();
|
||||
// Bob is not connected to the room but he is still in the rtc memberships declaring that
|
||||
// he is using that focus to publish, so he should still appear as a publisher
|
||||
expect(updatedPublishers?.length).toEqual(2);
|
||||
const pp = updatedPublishers?.find(
|
||||
(p) => p.membership.userId == "@bob:example.org",
|
||||
);
|
||||
expect(pp).toBeDefined();
|
||||
expect(pp!.participant).not.toBeDefined();
|
||||
expect(
|
||||
updatedPublishers?.some(
|
||||
(p) => p.participant?.identity === "@dan:example.org:DEV333",
|
||||
),
|
||||
).toBeTruthy();
|
||||
// Now if bob is not in the rtc memberships, he should disappear
|
||||
const noBob = rtcMemberships.filter(
|
||||
({ membership }) => membership.userId !== "@bob:example.org",
|
||||
);
|
||||
fakeMembershipsFocusMap$.next(noBob);
|
||||
expect(observedPublishers.pop()?.length).toEqual(1);
|
||||
});
|
||||
|
||||
it("should be scoped to parent scope", (): void => {
|
||||
setupTest();
|
||||
|
||||
const connection = setupRemoteConnection();
|
||||
|
||||
let observedPublishers: PublishingParticipant[][] = [];
|
||||
const s = connection.allLivekitParticipants$.subscribe((publishers) => {
|
||||
observedPublishers.push(publishers);
|
||||
});
|
||||
onTestFinished(() => s.unsubscribe());
|
||||
|
||||
let participants: RemoteParticipant[] = [
|
||||
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
|
||||
];
|
||||
|
||||
// Let's simulate 3 members on the livekitRoom
|
||||
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
|
||||
new Map(participants.map((p) => [p.identity, p])),
|
||||
);
|
||||
|
||||
for (const participant of participants) {
|
||||
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
|
||||
}
|
||||
|
||||
// At this point there should be no publishers
|
||||
expect(observedPublishers.pop()!.length).toEqual(0);
|
||||
|
||||
const rtcMemberships = [
|
||||
// Say bob is on the same focus
|
||||
{
|
||||
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
|
||||
transport: livekitFocus,
|
||||
},
|
||||
];
|
||||
// signal this change in rtc memberships
|
||||
fakeMembershipsFocusMap$.next(rtcMemberships);
|
||||
|
||||
// We should have bob has a publisher now
|
||||
const publishers = observedPublishers.pop();
|
||||
expect(publishers?.length).toEqual(1);
|
||||
expect(publishers?.[0].participant?.identity).toEqual(
|
||||
"@bob:example.org:DEV111",
|
||||
);
|
||||
|
||||
// end the parent scope
|
||||
testScope.end();
|
||||
observedPublishers = [];
|
||||
|
||||
// SHOULD NOT emit any more publishers as the scope is ended
|
||||
participants = participants.filter(
|
||||
(p) => p.identity !== "@bob:example.org:DEV111",
|
||||
);
|
||||
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
|
||||
new Map(participants.map((p) => [p.identity, p])),
|
||||
);
|
||||
fakeRoomEventEmiter.emit(
|
||||
RoomEvent.ParticipantDisconnected,
|
||||
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
|
||||
);
|
||||
|
||||
expect(observedPublishers.length).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PublishConnection", () => {
|
||||
// let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
|
||||
let roomFactoryMock: Mock<() => LivekitRoom>;
|
||||
let muteStates: MockedObject<MuteStates>;
|
||||
|
||||
function setUpPublishConnection(): void {
|
||||
setupTest();
|
||||
|
||||
roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
|
||||
|
||||
muteStates = mockMuteStates();
|
||||
|
||||
// fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
|
||||
// name: "BackgroundBlur",
|
||||
// restart: vi.fn().mockResolvedValue(undefined),
|
||||
// setOptions: vi.fn().mockResolvedValue(undefined),
|
||||
// getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
|
||||
// isRunning: vi.fn().mockReturnValue(false)
|
||||
// });
|
||||
}
|
||||
|
||||
describe("Livekit room creation", () => {
|
||||
function createSetup(): void {
|
||||
setUpPublishConnection();
|
||||
|
||||
const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
});
|
||||
|
||||
const opts: ConnectionOpts = {
|
||||
client: client,
|
||||
transport: livekitFocus,
|
||||
remoteTransports$: fakeMembershipsFocusMap$,
|
||||
scope: testScope,
|
||||
livekitRoomFactory: roomFactoryMock,
|
||||
};
|
||||
|
||||
const audioInput = {
|
||||
available$: of(new Map([["mic1", { id: "mic1" }]])),
|
||||
selected$: new BehaviorSubject({ id: "mic1" }),
|
||||
select(): void {},
|
||||
};
|
||||
|
||||
const videoInput = {
|
||||
available$: of(new Map([["cam1", { id: "cam1" }]])),
|
||||
selected$: new BehaviorSubject({ id: "cam1" }),
|
||||
select(): void {},
|
||||
};
|
||||
|
||||
const audioOutput = {
|
||||
available$: of(new Map([["speaker", { id: "speaker" }]])),
|
||||
selected$: new BehaviorSubject({ id: "speaker" }),
|
||||
select(): void {},
|
||||
};
|
||||
|
||||
// TODO understand what is wrong with our mocking that requires ts-expect-error
|
||||
const fakeDevices = mockMediaDevices({
|
||||
// @ts-expect-error Mocking only
|
||||
audioInput,
|
||||
// @ts-expect-error Mocking only
|
||||
videoInput,
|
||||
// @ts-expect-error Mocking only
|
||||
audioOutput,
|
||||
});
|
||||
|
||||
new PublishConnection(
|
||||
opts,
|
||||
fakeDevices,
|
||||
muteStates,
|
||||
undefined,
|
||||
fakeTrackProcessorSubject$,
|
||||
);
|
||||
}
|
||||
|
||||
it("should create room with proper initial audio and video settings", () => {
|
||||
createSetup();
|
||||
|
||||
expect(roomFactoryMock).toHaveBeenCalled();
|
||||
|
||||
const lastCallArgs =
|
||||
roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
|
||||
|
||||
const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
|
||||
expect(roomOptions).toBeDefined();
|
||||
|
||||
expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
|
||||
expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
|
||||
expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
|
||||
});
|
||||
|
||||
it("respect controlledAudioDevices", () => {
|
||||
// TODO: Refactor the code to make it testable.
|
||||
// The UrlParams module is a singleton has a cache and is very hard to test.
|
||||
// This breaks other tests as well if not handled properly.
|
||||
// vi.mock(import("./../UrlParams"), () => {
|
||||
// return {
|
||||
// getUrlParams: vi.fn().mockReturnValue({
|
||||
// controlledAudioDevices: true
|
||||
// })
|
||||
// };
|
||||
// });
|
||||
});
|
||||
});
|
||||
});
|
||||
226
src/state/CallViewModel/remoteMembers/Connection.ts
Normal file
226
src/state/CallViewModel/remoteMembers/Connection.ts
Normal file
@@ -0,0 +1,226 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
connectedParticipantsObserver,
|
||||
connectionStateObserver,
|
||||
} from "@livekit/components-core";
|
||||
import {
|
||||
ConnectionError,
|
||||
type ConnectionState as LivekitConenctionState,
|
||||
type Room as LivekitRoom,
|
||||
type LocalParticipant,
|
||||
type RemoteParticipant,
|
||||
RoomEvent,
|
||||
} from "livekit-client";
|
||||
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { BehaviorSubject, type Observable } from "rxjs";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
getSFUConfigWithOpenID,
|
||||
type OpenIDClientParts,
|
||||
type SFUConfig,
|
||||
} from "../../../livekit/openIDSFU.ts";
|
||||
import { type Behavior } from "../../Behavior.ts";
|
||||
import { type ObservableScope } from "../../ObservableScope.ts";
|
||||
import {
|
||||
InsufficientCapacityError,
|
||||
SFURoomCreationRestrictedError,
|
||||
} from "../../../utils/errors.ts";
|
||||
|
||||
export type PublishingParticipant = LocalParticipant | RemoteParticipant;
|
||||
|
||||
export interface ConnectionOpts {
|
||||
/** The media transport to connect to. */
|
||||
transport: LivekitTransport;
|
||||
/** The Matrix client to use for OpenID and SFU config requests. */
|
||||
client: OpenIDClientParts;
|
||||
/** The observable scope to use for this connection. */
|
||||
scope: ObservableScope;
|
||||
|
||||
/** Optional factory to create the LiveKit room, mainly for testing purposes. */
|
||||
livekitRoomFactory: () => LivekitRoom;
|
||||
}
|
||||
|
||||
export type ConnectionState =
|
||||
| { state: "Initialized" }
|
||||
| { state: "FetchingConfig"; transport: LivekitTransport }
|
||||
| { state: "ConnectingToLkRoom"; transport: LivekitTransport }
|
||||
| { state: "PublishingTracks"; transport: LivekitTransport }
|
||||
| { state: "FailedToStart"; error: Error; transport: LivekitTransport }
|
||||
| {
|
||||
state: "ConnectedToLkRoom";
|
||||
livekitConnectionState$: Observable<LivekitConenctionState>;
|
||||
transport: LivekitTransport;
|
||||
}
|
||||
| { state: "Stopped"; transport: LivekitTransport };
|
||||
|
||||
/**
|
||||
* A connection to a Matrix RTC LiveKit backend.
|
||||
*
|
||||
* Expose observables for participants and connection state.
|
||||
*/
|
||||
export class Connection {
|
||||
// Private Behavior
|
||||
private readonly _state$ = new BehaviorSubject<ConnectionState>({
|
||||
state: "Initialized",
|
||||
});
|
||||
|
||||
/**
|
||||
* The current state of the connection to the media transport.
|
||||
*/
|
||||
public readonly state$: Behavior<ConnectionState> = this._state$;
|
||||
|
||||
/**
|
||||
* Whether the connection has been stopped.
|
||||
* @see Connection.stop
|
||||
* */
|
||||
protected stopped = false;
|
||||
|
||||
/**
|
||||
* Starts the connection.
|
||||
*
|
||||
* This will:
|
||||
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
|
||||
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
|
||||
* 3. Connect to the configured LiveKit room.
|
||||
*
|
||||
* The errors are also represented as a state in the `state$` observable.
|
||||
* It is safe to ignore those errors and handle them accordingly via the `state$` observable.
|
||||
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
|
||||
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
|
||||
*/
|
||||
// TODO dont make this throw and instead store a connection error state in this class?
|
||||
// TODO consider an autostart pattern...
|
||||
public async start(): Promise<void> {
|
||||
this.stopped = false;
|
||||
try {
|
||||
this._state$.next({
|
||||
state: "FetchingConfig",
|
||||
transport: this.transport,
|
||||
});
|
||||
const { url, jwt } = await this.getSFUConfigWithOpenID();
|
||||
// If we were stopped while fetching the config, don't proceed to connect
|
||||
if (this.stopped) return;
|
||||
|
||||
this._state$.next({
|
||||
state: "ConnectingToLkRoom",
|
||||
transport: this.transport,
|
||||
});
|
||||
try {
|
||||
await this.livekitRoom.connect(url, jwt);
|
||||
} catch (e) {
|
||||
// LiveKit uses 503 to indicate that the server has hit its track limits.
|
||||
// https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
|
||||
// It also errors with a status code of 200 (yes, really) for room
|
||||
// participant limits.
|
||||
// LiveKit Cloud uses 429 for connection limits.
|
||||
// Either way, all these errors can be explained as "insufficient capacity".
|
||||
if (e instanceof ConnectionError) {
|
||||
if (e.status === 503 || e.status === 200 || e.status === 429) {
|
||||
throw new InsufficientCapacityError();
|
||||
}
|
||||
if (e.status === 404) {
|
||||
// error msg is "Could not establish signal connection: requested room does not exist"
|
||||
// The room does not exist. There are two different modes of operation for the SFU:
|
||||
// - the room is created on the fly when connecting (livekit `auto_create` option)
|
||||
// - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
|
||||
// In the first case there will not be a 404, so we are in the second case.
|
||||
throw new SFURoomCreationRestrictedError();
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
// If we were stopped while connecting, don't proceed to update state.
|
||||
if (this.stopped) return;
|
||||
|
||||
this._state$.next({
|
||||
state: "ConnectedToLkRoom",
|
||||
transport: this.transport,
|
||||
livekitConnectionState$: connectionStateObserver(this.livekitRoom),
|
||||
});
|
||||
} catch (error) {
|
||||
this._state$.next({
|
||||
state: "FailedToStart",
|
||||
error: error instanceof Error ? error : new Error(`${error}`),
|
||||
transport: this.transport,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
|
||||
return await getSFUConfigWithOpenID(
|
||||
this.client,
|
||||
this.transport.livekit_service_url,
|
||||
this.transport.livekit_alias,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the connection.
|
||||
*
|
||||
* This will disconnect from the LiveKit room.
|
||||
* If the connection is already stopped, this is a no-op.
|
||||
*/
|
||||
public async stop(): Promise<void> {
|
||||
if (this.stopped) return;
|
||||
await this.livekitRoom.disconnect();
|
||||
this._state$.next({
|
||||
state: "Stopped",
|
||||
transport: this.transport,
|
||||
});
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* An observable of the participants that are publishing on this connection.
|
||||
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
|
||||
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
|
||||
*/
|
||||
|
||||
public readonly participantsWithTrack$: Behavior<PublishingParticipant[]>;
|
||||
|
||||
/**
|
||||
* The media transport to connect to.
|
||||
*/
|
||||
public readonly transport: LivekitTransport;
|
||||
|
||||
private readonly client: OpenIDClientParts;
|
||||
public readonly livekitRoom: LivekitRoom;
|
||||
|
||||
/**
|
||||
* Creates a new connection to a matrix RTC LiveKit backend.
|
||||
*
|
||||
* @param livekitRoom - LiveKit room instance to use.
|
||||
* @param opts - Connection options {@link ConnectionOpts}.
|
||||
*
|
||||
*/
|
||||
public constructor(opts: ConnectionOpts, logger?: Logger) {
|
||||
logger?.info(
|
||||
`[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
|
||||
);
|
||||
const { transport, client, scope } = opts;
|
||||
|
||||
this.livekitRoom = opts.livekitRoomFactory();
|
||||
this.transport = transport;
|
||||
this.client = client;
|
||||
|
||||
this.participantsWithTrack$ = scope.behavior(
|
||||
connectedParticipantsObserver(this.livekitRoom, {
|
||||
additionalRoomEvents: [
|
||||
RoomEvent.TrackPublished,
|
||||
RoomEvent.TrackUnpublished,
|
||||
],
|
||||
}),
|
||||
[],
|
||||
);
|
||||
|
||||
scope.onEnd(() => void this.stop());
|
||||
}
|
||||
}
|
||||
114
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
Normal file
114
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import {
|
||||
type E2EEOptions,
|
||||
Room as LivekitRoom,
|
||||
type RoomOptions,
|
||||
} from "livekit-client";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type ObservableScope } from "../../ObservableScope.ts";
|
||||
import { Connection } from "./Connection.ts";
|
||||
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
|
||||
import type { MediaDevices } from "../../MediaDevices.ts";
|
||||
import type { Behavior } from "../../Behavior.ts";
|
||||
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import { defaultLiveKitOptions } from "../../../livekit/options.ts";
|
||||
|
||||
export interface ConnectionFactory {
|
||||
createConnection(
|
||||
transport: LivekitTransport,
|
||||
scope: ObservableScope,
|
||||
logger: Logger,
|
||||
): Connection;
|
||||
}
|
||||
|
||||
export class ECConnectionFactory implements ConnectionFactory {
|
||||
private readonly livekitRoomFactory: () => LivekitRoom;
|
||||
|
||||
/**
|
||||
* Creates a ConnectionFactory for LiveKit connections.
|
||||
*
|
||||
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
|
||||
* @param devices - Used for video/audio out/in capture options.
|
||||
* @param processorState$ - Effects like background blur (only for publishing connection?)
|
||||
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room.
|
||||
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
|
||||
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
|
||||
*/
|
||||
public constructor(
|
||||
private client: OpenIDClientParts,
|
||||
private devices: MediaDevices,
|
||||
private processorState$: Behavior<ProcessorState>,
|
||||
private e2eeLivekitOptions: E2EEOptions | undefined,
|
||||
private controlledAudioDevices: boolean,
|
||||
livekitRoomFactory?: () => LivekitRoom,
|
||||
) {
|
||||
const defaultFactory = (): LivekitRoom =>
|
||||
new LivekitRoom(
|
||||
generateRoomOption(
|
||||
this.devices,
|
||||
this.processorState$.value,
|
||||
this.e2eeLivekitOptions,
|
||||
this.controlledAudioDevices,
|
||||
),
|
||||
);
|
||||
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
|
||||
}
|
||||
|
||||
public createConnection(
|
||||
transport: LivekitTransport,
|
||||
scope: ObservableScope,
|
||||
logger: Logger,
|
||||
): Connection {
|
||||
return new Connection(
|
||||
{
|
||||
transport,
|
||||
client: this.client,
|
||||
scope: scope,
|
||||
livekitRoomFactory: this.livekitRoomFactory,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
|
||||
*/
|
||||
function generateRoomOption(
|
||||
devices: MediaDevices,
|
||||
processorState: ProcessorState,
|
||||
e2eeLivekitOptions: E2EEOptions | undefined,
|
||||
controlledAudioDevices: boolean,
|
||||
): RoomOptions {
|
||||
return {
|
||||
...defaultLiveKitOptions,
|
||||
videoCaptureDefaults: {
|
||||
...defaultLiveKitOptions.videoCaptureDefaults,
|
||||
deviceId: devices.videoInput.selected$.value?.id,
|
||||
processor: processorState.processor,
|
||||
},
|
||||
audioCaptureDefaults: {
|
||||
...defaultLiveKitOptions.audioCaptureDefaults,
|
||||
deviceId: devices.audioInput.selected$.value?.id,
|
||||
},
|
||||
audioOutput: {
|
||||
// When using controlled audio devices, we don't want to set the
|
||||
// deviceId here, because it will be set by the native app.
|
||||
// (also the id does not need to match a browser device id)
|
||||
deviceId: controlledAudioDevices
|
||||
? undefined
|
||||
: devices.audioOutput.selected$.value?.id,
|
||||
},
|
||||
e2ee: e2eeLivekitOptions,
|
||||
// TODO test and consider this:
|
||||
// webAudioMix: true,
|
||||
};
|
||||
}
|
||||
297
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
Normal file
297
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
Normal file
@@ -0,0 +1,297 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Participant as LivekitParticipant } from "livekit-client";
|
||||
|
||||
import { ObservableScope } from "../../ObservableScope.ts";
|
||||
import {
|
||||
type IConnectionManager,
|
||||
createConnectionManager$,
|
||||
} from "./ConnectionManager.ts";
|
||||
import { type ConnectionFactory } from "./ConnectionFactory.ts";
|
||||
import { type Connection } from "./Connection.ts";
|
||||
import { flushPromises, withTestScheduler } from "../../../utils/test.ts";
|
||||
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
|
||||
|
||||
// Some test constants
|
||||
|
||||
const TRANSPORT_1: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.example.org",
|
||||
livekit_alias: "!alias:example.org",
|
||||
};
|
||||
|
||||
const TRANSPORT_2: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.sample.com",
|
||||
livekit_alias: "!alias:sample.com",
|
||||
};
|
||||
|
||||
// const TRANSPORT_3: LivekitTransport = {
|
||||
// type: "livekit",
|
||||
// livekit_service_url: "https://lk-other.sample.com",
|
||||
// livekit_alias: "!alias:sample.com",
|
||||
// };
|
||||
let fakeConnectionFactory: ConnectionFactory;
|
||||
let testScope: ObservableScope;
|
||||
let testTransportStream$: BehaviorSubject<LivekitTransport[]>;
|
||||
let connectionManagerInputs: {
|
||||
scope: ObservableScope;
|
||||
connectionFactory: ConnectionFactory;
|
||||
inputTransports$: BehaviorSubject<LivekitTransport[]>;
|
||||
};
|
||||
let manager: IConnectionManager;
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
|
||||
fakeConnectionFactory = {} as unknown as ConnectionFactory;
|
||||
vi.mocked(fakeConnectionFactory).createConnection = vi
|
||||
.fn()
|
||||
.mockImplementation(
|
||||
(transport: LivekitTransport, scope: ObservableScope) => {
|
||||
const mockConnection = {
|
||||
transport,
|
||||
} as unknown as Connection;
|
||||
vi.mocked(mockConnection).start = vi.fn();
|
||||
vi.mocked(mockConnection).stop = vi.fn();
|
||||
// Tie the connection's lifecycle to the scope to test scope lifecycle management
|
||||
scope.onEnd(() => {
|
||||
void mockConnection.stop();
|
||||
});
|
||||
return mockConnection;
|
||||
},
|
||||
);
|
||||
|
||||
testTransportStream$ = new BehaviorSubject<LivekitTransport[]>([]);
|
||||
connectionManagerInputs = {
|
||||
scope: testScope,
|
||||
connectionFactory: fakeConnectionFactory,
|
||||
inputTransports$: testTransportStream$,
|
||||
};
|
||||
manager = createConnectionManager$(connectionManagerInputs);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testScope.end();
|
||||
});
|
||||
|
||||
describe("connections$ stream", () => {
|
||||
test("Should create and start new connections for each transports", async () => {
|
||||
const managedConnections = Promise.withResolvers<Connection[]>();
|
||||
manager.connections$.subscribe((connections) => {
|
||||
if (connections.length > 0) managedConnections.resolve(connections);
|
||||
});
|
||||
|
||||
connectionManagerInputs.inputTransports$.next([TRANSPORT_1, TRANSPORT_2]);
|
||||
|
||||
const connections = await managedConnections.promise;
|
||||
|
||||
expect(connections.length).toBe(2);
|
||||
|
||||
expect(
|
||||
vi.mocked(fakeConnectionFactory).createConnection,
|
||||
).toHaveBeenCalledTimes(2);
|
||||
|
||||
const conn1 = connections.find((c) =>
|
||||
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
|
||||
);
|
||||
expect(conn1).toBeDefined();
|
||||
expect(conn1!.start).toHaveBeenCalled();
|
||||
|
||||
const conn2 = connections.find((c) =>
|
||||
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
|
||||
);
|
||||
expect(conn2).toBeDefined();
|
||||
expect(conn2!.start).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("Should start connection only once", async () => {
|
||||
const observedConnections: Connection[][] = [];
|
||||
manager.connections$.subscribe((connections) => {
|
||||
observedConnections.push(connections);
|
||||
});
|
||||
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]);
|
||||
|
||||
await flushPromises();
|
||||
const connections = observedConnections.pop()!;
|
||||
|
||||
expect(connections.length).toBe(2);
|
||||
expect(
|
||||
vi.mocked(fakeConnectionFactory).createConnection,
|
||||
).toHaveBeenCalledTimes(2);
|
||||
|
||||
const conn2 = connections.find((c) =>
|
||||
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
|
||||
);
|
||||
expect(conn2).toBeDefined();
|
||||
|
||||
const conn1 = connections.find((c) =>
|
||||
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
|
||||
);
|
||||
expect(conn1).toBeDefined();
|
||||
expect(conn1!.start).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("Should cleanup connections when not needed anymore", async () => {
|
||||
const observedConnections: Connection[][] = [];
|
||||
manager.connections$.subscribe((connections) => {
|
||||
observedConnections.push(connections);
|
||||
});
|
||||
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]);
|
||||
|
||||
await flushPromises();
|
||||
|
||||
const conn2 = observedConnections
|
||||
.pop()!
|
||||
.find((c) => areLivekitTransportsEqual(c.transport, TRANSPORT_2))!;
|
||||
|
||||
testTransportStream$.next([TRANSPORT_1]);
|
||||
|
||||
await flushPromises();
|
||||
|
||||
// The second connection should have been stopped has it is no longer needed
|
||||
expect(conn2.stop).toHaveBeenCalled();
|
||||
|
||||
// The first connection should still be active
|
||||
const conn1 = observedConnections.pop()![0];
|
||||
expect(conn1.stop).not.toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
describe("connectionManagerData$ stream", () => {
|
||||
// Used in test to control fake connections' participantsWithTrack$ streams
|
||||
let fakePublishingParticipantsStreams: Map<
|
||||
string,
|
||||
BehaviorSubject<LivekitParticipant[]>
|
||||
>;
|
||||
|
||||
function keyForTransport(transport: LivekitTransport): string {
|
||||
return `${transport.livekit_service_url}|${transport.livekit_alias}`;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
fakePublishingParticipantsStreams = new Map();
|
||||
// need a more advanced fake connection factory
|
||||
vi.mocked(fakeConnectionFactory).createConnection = vi
|
||||
.fn()
|
||||
.mockImplementation(
|
||||
(transport: LivekitTransport, scope: ObservableScope) => {
|
||||
const fakePublishingParticipants$ = new BehaviorSubject<
|
||||
LivekitParticipant[]
|
||||
>([]);
|
||||
const mockConnection = {
|
||||
transport,
|
||||
participantsWithTrack$: fakePublishingParticipants$,
|
||||
} as unknown as Connection;
|
||||
vi.mocked(mockConnection).start = vi.fn();
|
||||
vi.mocked(mockConnection).stop = vi.fn();
|
||||
// Tie the connection's lifecycle to the scope to test scope lifecycle management
|
||||
scope.onEnd(() => {
|
||||
void mockConnection.stop();
|
||||
});
|
||||
|
||||
fakePublishingParticipantsStreams.set(
|
||||
keyForTransport(transport),
|
||||
fakePublishingParticipants$,
|
||||
);
|
||||
return mockConnection;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("Should report connections with the publishing participants", () => {
|
||||
withTestScheduler(({ expectObservable, schedule, behavior }) => {
|
||||
manager = createConnectionManager$({
|
||||
...connectionManagerInputs,
|
||||
inputTransports$: behavior("a", {
|
||||
a: [TRANSPORT_1, TRANSPORT_2],
|
||||
}),
|
||||
});
|
||||
|
||||
const conn1Participants$ = fakePublishingParticipantsStreams.get(
|
||||
keyForTransport(TRANSPORT_1),
|
||||
)!;
|
||||
|
||||
schedule("-a-b", {
|
||||
a: () => {
|
||||
conn1Participants$.next([
|
||||
{ identity: "user1A" } as LivekitParticipant,
|
||||
]);
|
||||
},
|
||||
b: () => {
|
||||
conn1Participants$.next([
|
||||
{ identity: "user1A" } as LivekitParticipant,
|
||||
{ identity: "user1B" } as LivekitParticipant,
|
||||
]);
|
||||
},
|
||||
});
|
||||
|
||||
const conn2Participants$ = fakePublishingParticipantsStreams.get(
|
||||
keyForTransport(TRANSPORT_2),
|
||||
)!;
|
||||
|
||||
schedule("--a", {
|
||||
a: () => {
|
||||
conn2Participants$.next([
|
||||
{ identity: "user2A" } as LivekitParticipant,
|
||||
]);
|
||||
},
|
||||
});
|
||||
|
||||
expectObservable(manager.connectionManagerData$).toBe("abcd", {
|
||||
a: expect.toSatisfy((data) => {
|
||||
return (
|
||||
data.getConnections().length == 2 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1).length == 0 &&
|
||||
data.getParticipantForTransport(TRANSPORT_2).length == 0
|
||||
);
|
||||
}),
|
||||
b: expect.toSatisfy((data) => {
|
||||
return (
|
||||
data.getConnections().length == 2 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1).length == 1 &&
|
||||
data.getParticipantForTransport(TRANSPORT_2).length == 0 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A"
|
||||
);
|
||||
}),
|
||||
c: expect.toSatisfy((data) => {
|
||||
return (
|
||||
data.getConnections().length == 2 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1).length == 1 &&
|
||||
data.getParticipantForTransport(TRANSPORT_2).length == 1 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1)[0].identity ==
|
||||
"user1A" &&
|
||||
data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A"
|
||||
);
|
||||
}),
|
||||
d: expect.toSatisfy((data) => {
|
||||
return (
|
||||
data.getConnections().length == 2 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1).length == 2 &&
|
||||
data.getParticipantForTransport(TRANSPORT_2).length == 1 &&
|
||||
data.getParticipantForTransport(TRANSPORT_1)[0].identity ==
|
||||
"user1A" &&
|
||||
data.getParticipantForTransport(TRANSPORT_1)[1].identity ==
|
||||
"user1B" &&
|
||||
data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A"
|
||||
);
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
222
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
Normal file
222
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
Normal file
@@ -0,0 +1,222 @@
|
||||
// TODOs:
|
||||
// - make ConnectionManager its own actual class
|
||||
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LivekitTransport,
|
||||
type ParticipantId,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
|
||||
|
||||
import { type Behavior } from "../../Behavior.ts";
|
||||
import { type Connection } from "./Connection.ts";
|
||||
import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
|
||||
import { generateKeyed$ } from "../../../utils/observable.ts";
|
||||
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
|
||||
import { type ConnectionFactory } from "./ConnectionFactory.ts";
|
||||
|
||||
export class ConnectionManagerData {
|
||||
private readonly store: Map<
|
||||
string,
|
||||
[Connection, (LocalParticipant | RemoteParticipant)[]]
|
||||
> = new Map();
|
||||
|
||||
public constructor() {}
|
||||
|
||||
public add(
|
||||
connection: Connection,
|
||||
participants: (LocalParticipant | RemoteParticipant)[],
|
||||
): void {
|
||||
const key = this.getKey(connection.transport);
|
||||
const existing = this.store.get(key);
|
||||
if (!existing) {
|
||||
this.store.set(key, [connection, participants]);
|
||||
} else {
|
||||
existing[1].push(...participants);
|
||||
}
|
||||
}
|
||||
|
||||
private getKey(transport: LivekitTransport): string {
|
||||
return transport.livekit_service_url + "|" + transport.livekit_alias;
|
||||
}
|
||||
|
||||
public getConnections(): Connection[] {
|
||||
return Array.from(this.store.values()).map(([connection]) => connection);
|
||||
}
|
||||
|
||||
public getConnectionForTransport(
|
||||
transport: LivekitTransport,
|
||||
): Connection | undefined {
|
||||
return this.store.get(this.getKey(transport))?.[0];
|
||||
}
|
||||
|
||||
public getParticipantForTransport(
|
||||
transport: LivekitTransport,
|
||||
): (LocalParticipant | RemoteParticipant)[] {
|
||||
const key = transport.livekit_service_url + "|" + transport.livekit_alias;
|
||||
const existing = this.store.get(key);
|
||||
if (existing) {
|
||||
return existing[1];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
/**
|
||||
* Get all connections where the given participant is publishing.
|
||||
* In theory, there could be several connections where the same participant is publishing but with
|
||||
* only well behaving clients a participant should only be publishing on a single connection.
|
||||
* @param participantId
|
||||
*/
|
||||
public getConnectionsForParticipant(
|
||||
participantId: ParticipantId,
|
||||
): Connection[] {
|
||||
const connections: Connection[] = [];
|
||||
for (const [connection, participants] of this.store.values()) {
|
||||
if (participants.some((p) => p.identity === participantId)) {
|
||||
connections.push(connection);
|
||||
}
|
||||
}
|
||||
return connections;
|
||||
}
|
||||
}
|
||||
interface Props {
|
||||
scope: ObservableScope;
|
||||
connectionFactory: ConnectionFactory;
|
||||
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
|
||||
}
|
||||
// TODO - write test for scopes (do we really need to bind scope)
|
||||
export interface IConnectionManager {
|
||||
transports$: Behavior<Epoch<LivekitTransport[]>>;
|
||||
connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
|
||||
connections$: Behavior<Epoch<Connection[]>>;
|
||||
}
|
||||
/**
|
||||
* Crete a `ConnectionManager`
|
||||
* @param scope the observable scope used by this object.
|
||||
* @param connectionFactory used to create new connections.
|
||||
* @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
|
||||
* Each of these behaviors can be interpreted as subscribed list of transports.
|
||||
*
|
||||
* Using `registerTransports` independent external modules can control what connections
|
||||
* are created by the ConnectionManager.
|
||||
*
|
||||
* The connection manager will remove all duplicate transports in each subscibed list.
|
||||
*
|
||||
* See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe.
|
||||
*/
|
||||
export function createConnectionManager$({
|
||||
scope,
|
||||
connectionFactory,
|
||||
inputTransports$,
|
||||
}: Props): IConnectionManager {
|
||||
const logger = rootLogger.getChild("ConnectionManager");
|
||||
|
||||
const running$ = new BehaviorSubject(true);
|
||||
scope.onEnd(() => running$.next(false));
|
||||
// TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
|
||||
|
||||
/**
|
||||
* All transports currently managed by the ConnectionManager.
|
||||
*
|
||||
* This list does not include duplicate transports.
|
||||
*
|
||||
* It is build based on the list of subscribed transports (`transportsSubscriptions$`).
|
||||
* externally this is modified via `registerTransports()`.
|
||||
*/
|
||||
const transports$ = scope.behavior(
|
||||
combineLatest([running$, inputTransports$]).pipe(
|
||||
map(([running, transports]) =>
|
||||
transports.mapInner((transport) => (running ? transport : [])),
|
||||
),
|
||||
map((transports) => transports.mapInner(removeDuplicateTransports)),
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* Connections for each transport in use by one or more session members.
|
||||
*/
|
||||
const connections$ = scope.behavior(
|
||||
generateKeyed$<Epoch<LivekitTransport[]>, Connection, Epoch<Connection[]>>(
|
||||
transports$,
|
||||
(transports, createOrGet) => {
|
||||
const createConnection =
|
||||
(
|
||||
transport: LivekitTransport,
|
||||
): ((scope: ObservableScope) => Connection) =>
|
||||
(scope) => {
|
||||
const connection = connectionFactory.createConnection(
|
||||
transport,
|
||||
scope,
|
||||
logger,
|
||||
);
|
||||
// Start the connection immediately
|
||||
// Use connection state to track connection progress
|
||||
void connection.start();
|
||||
// TODO subscribe to connection state to retry or log issues?
|
||||
return connection;
|
||||
};
|
||||
|
||||
return transports.mapInner((transports) => {
|
||||
return transports.map((transport) => {
|
||||
const key =
|
||||
transport.livekit_service_url + "|" + transport.livekit_alias;
|
||||
return createOrGet(key, createConnection(transport));
|
||||
});
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const connectionManagerData$ = scope.behavior(
|
||||
connections$.pipe(
|
||||
switchMap((connections) => {
|
||||
const epoch = connections.epoch;
|
||||
|
||||
// Map the connections to list of {connection, participants}[]
|
||||
const listOfConnectionsWithPublishingParticipants =
|
||||
connections.value.map((connection) => {
|
||||
return connection.participantsWithTrack$.pipe(
|
||||
map((participants) => ({
|
||||
connection,
|
||||
participants,
|
||||
})),
|
||||
);
|
||||
});
|
||||
|
||||
// combineLatest the several streams into a single stream with the ConnectionManagerData
|
||||
return combineLatest(listOfConnectionsWithPublishingParticipants).pipe(
|
||||
map(
|
||||
(lists) =>
|
||||
new Epoch(
|
||||
lists.reduce((data, { connection, participants }) => {
|
||||
data.add(connection, participants);
|
||||
return data;
|
||||
}, new ConnectionManagerData()),
|
||||
epoch,
|
||||
),
|
||||
),
|
||||
);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
return { transports$, connectionManagerData$, connections$ };
|
||||
}
|
||||
|
||||
function removeDuplicateTransports(
|
||||
transports: LivekitTransport[],
|
||||
): LivekitTransport[] {
|
||||
return transports.reduce((acc, transport) => {
|
||||
if (!acc.some((t) => areLivekitTransportsEqual(t, transport)))
|
||||
acc.push(transport);
|
||||
return acc;
|
||||
}, [] as LivekitTransport[]);
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, test, vi, expect, beforeEach, afterEach } from "vitest";
|
||||
import {
|
||||
type CallMembership,
|
||||
type LivekitTransport,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
|
||||
import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils";
|
||||
|
||||
import { type IConnectionManager } from "./ConnectionManager.ts";
|
||||
import {
|
||||
type MatrixLivekitMember,
|
||||
createMatrixLivekitMembers$,
|
||||
areLivekitTransportsEqual,
|
||||
} from "./MatrixLivekitMembers.ts";
|
||||
import { ObservableScope } from "../../ObservableScope.ts";
|
||||
import { ConnectionManagerData } from "./ConnectionManager.ts";
|
||||
import {
|
||||
mockCallMembership,
|
||||
mockRemoteParticipant,
|
||||
type OurRunHelpers,
|
||||
withTestScheduler,
|
||||
} from "../../../utils/test.ts";
|
||||
import { type Connection } from "./Connection.ts";
|
||||
|
||||
let testScope: ObservableScope;
|
||||
let mockMatrixRoom: MatrixRoom;
|
||||
|
||||
// The merger beeing tested
|
||||
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
mockMatrixRoom = vi.mocked<MatrixRoom>({
|
||||
getMember: vi.fn().mockImplementation((userId: string) => {
|
||||
return {
|
||||
userId,
|
||||
rawDisplayName: userId.replace("@", "").replace(":example.org", ""),
|
||||
getMxcAvatarUrl: vi.fn().mockReturnValue(null),
|
||||
} as unknown as RoomMember;
|
||||
}),
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
} as unknown as MatrixRoom);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testScope.end();
|
||||
});
|
||||
|
||||
test("should signal participant not yet connected to livekit", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const bobMembership = {
|
||||
userId: "@bob:example.org",
|
||||
deviceId: "DEV000",
|
||||
transports: [
|
||||
{
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.example.org",
|
||||
livekit_alias: "!alias:example.org",
|
||||
},
|
||||
],
|
||||
} as unknown as CallMembership;
|
||||
|
||||
const matrixLivekitMember$ = createMatrixLivekitMembers$({
|
||||
scope: testScope,
|
||||
membershipsWithTransport$: behavior("a", {
|
||||
a: [
|
||||
{
|
||||
membership: bobMembership,
|
||||
},
|
||||
],
|
||||
}),
|
||||
connectionManager: {
|
||||
connectionManagerData$: behavior("a", {
|
||||
a: new ConnectionManagerData(),
|
||||
}),
|
||||
transports$: behavior("a", { a: [] }),
|
||||
connections$: behavior("a", { a: [] }),
|
||||
},
|
||||
matrixRoom: mockMatrixRoom,
|
||||
});
|
||||
|
||||
expectObservable(matrixLivekitMember$).toBe("a", {
|
||||
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
|
||||
return (
|
||||
data.length == 1 &&
|
||||
data[0].membership === bobMembership &&
|
||||
data[0].participant === undefined &&
|
||||
data[0].connection === undefined
|
||||
);
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function aConnectionManager(
|
||||
data: ConnectionManagerData,
|
||||
behavior: OurRunHelpers["behavior"],
|
||||
): IConnectionManager {
|
||||
return {
|
||||
connectionManagerData$: behavior("a", { a: data }),
|
||||
transports$: behavior("a", {
|
||||
a: data.getConnections().map((connection) => connection.transport),
|
||||
}),
|
||||
connections$: behavior("a", { a: data.getConnections() }),
|
||||
};
|
||||
}
|
||||
|
||||
test("should signal participant on a connection that is publishing", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const transport: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.example.org",
|
||||
livekit_alias: "!alias:example.org",
|
||||
};
|
||||
|
||||
const bobMembership = mockCallMembership(
|
||||
"@bob:example.org",
|
||||
"DEV000",
|
||||
transport,
|
||||
);
|
||||
|
||||
const connectionWithPublisher = new ConnectionManagerData();
|
||||
const bobParticipantId = getParticipantId(
|
||||
bobMembership.userId,
|
||||
bobMembership.deviceId,
|
||||
);
|
||||
const connection = {
|
||||
transport: transport,
|
||||
} as unknown as Connection;
|
||||
connectionWithPublisher.add(connection, [
|
||||
mockRemoteParticipant({ identity: bobParticipantId }),
|
||||
]);
|
||||
const matrixLivekitMember$ = createMatrixLivekitMembers$({
|
||||
scope: testScope,
|
||||
membershipsWithTransport$: behavior("a", {
|
||||
a: [
|
||||
{
|
||||
membership: bobMembership,
|
||||
transport,
|
||||
},
|
||||
],
|
||||
}),
|
||||
connectionManager: aConnectionManager(connectionWithPublisher, behavior),
|
||||
matrixRoom: mockMatrixRoom,
|
||||
});
|
||||
|
||||
expectObservable(matrixLivekitMember$).toBe("a", {
|
||||
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
|
||||
expect(data.length).toEqual(1);
|
||||
expect(data[0].participant).toBeDefined();
|
||||
expect(data[0].connection).toBeDefined();
|
||||
expect(data[0].membership).toEqual(bobMembership);
|
||||
expect(
|
||||
areLivekitTransportsEqual(data[0].connection!.transport, transport),
|
||||
).toBe(true);
|
||||
return true;
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("should signal participant on a connection that is not publishing", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const transport: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.example.org",
|
||||
livekit_alias: "!alias:example.org",
|
||||
};
|
||||
|
||||
const bobMembership = mockCallMembership(
|
||||
"@bob:example.org",
|
||||
"DEV000",
|
||||
transport,
|
||||
);
|
||||
|
||||
const connectionWithPublisher = new ConnectionManagerData();
|
||||
// const bobParticipantId = getParticipantId(bobMembership.userId, bobMembership.deviceId);
|
||||
const connection = {
|
||||
transport: transport,
|
||||
} as unknown as Connection;
|
||||
connectionWithPublisher.add(connection, []);
|
||||
const matrixLivekitMember$ = createMatrixLivekitMembers$({
|
||||
scope: testScope,
|
||||
membershipsWithTransport$: behavior("a", {
|
||||
a: [
|
||||
{
|
||||
membership: bobMembership,
|
||||
transport,
|
||||
},
|
||||
],
|
||||
}),
|
||||
connectionManager: aConnectionManager(connectionWithPublisher, behavior),
|
||||
matrixRoom: mockMatrixRoom,
|
||||
});
|
||||
|
||||
expectObservable(matrixLivekitMember$).toBe("a", {
|
||||
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
|
||||
expect(data.length).toEqual(1);
|
||||
expect(data[0].participant).not.toBeDefined();
|
||||
expect(data[0].connection).toBeDefined();
|
||||
expect(data[0].membership).toEqual(bobMembership);
|
||||
expect(
|
||||
areLivekitTransportsEqual(data[0].connection!.transport, transport),
|
||||
).toBe(true);
|
||||
return true;
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Publication edge case", () => {
|
||||
test("bob is publishing in several connections", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const transportA: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.example.org",
|
||||
livekit_alias: "!alias:example.org",
|
||||
};
|
||||
|
||||
const transportB: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.sample.com",
|
||||
livekit_alias: "!alias:sample.com",
|
||||
};
|
||||
|
||||
const bobMembership = mockCallMembership(
|
||||
"@bob:example.org",
|
||||
"DEV000",
|
||||
transportA,
|
||||
);
|
||||
|
||||
const connectionWithPublisher = new ConnectionManagerData();
|
||||
const bobParticipantId = getParticipantId(
|
||||
bobMembership.userId,
|
||||
bobMembership.deviceId,
|
||||
);
|
||||
const connectionA = {
|
||||
transport: transportA,
|
||||
} as unknown as Connection;
|
||||
const connectionB = {
|
||||
transport: transportB,
|
||||
} as unknown as Connection;
|
||||
|
||||
connectionWithPublisher.add(connectionA, [
|
||||
mockRemoteParticipant({ identity: bobParticipantId }),
|
||||
]);
|
||||
connectionWithPublisher.add(connectionB, [
|
||||
mockRemoteParticipant({ identity: bobParticipantId }),
|
||||
]);
|
||||
const matrixLivekitMember$ = createMatrixLivekitMembers$({
|
||||
scope: testScope,
|
||||
membershipsWithTransport$: behavior("a", {
|
||||
a: [
|
||||
{
|
||||
membership: bobMembership,
|
||||
transport: transportA,
|
||||
},
|
||||
],
|
||||
}),
|
||||
connectionManager: aConnectionManager(
|
||||
connectionWithPublisher,
|
||||
behavior,
|
||||
),
|
||||
matrixRoom: mockMatrixRoom,
|
||||
});
|
||||
|
||||
expectObservable(matrixLivekitMember$).toBe("a", {
|
||||
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
|
||||
expect(data.length).toEqual(1);
|
||||
expect(data[0].participant).toBeDefined();
|
||||
expect(data[0].participant!.identity).toEqual(bobParticipantId);
|
||||
expect(data[0].connection).toBeDefined();
|
||||
expect(data[0].membership).toEqual(bobMembership);
|
||||
expect(
|
||||
areLivekitTransportsEqual(
|
||||
data[0].connection!.transport,
|
||||
transportA,
|
||||
),
|
||||
).toBe(true);
|
||||
return true;
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("bob is publishing in the wrong connection", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const transportA: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.example.org",
|
||||
livekit_alias: "!alias:example.org",
|
||||
};
|
||||
|
||||
const transportB: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.sample.com",
|
||||
livekit_alias: "!alias:sample.com",
|
||||
};
|
||||
|
||||
const bobMembership = mockCallMembership(
|
||||
"@bob:example.org",
|
||||
"DEV000",
|
||||
transportA,
|
||||
);
|
||||
|
||||
const connectionWithPublisher = new ConnectionManagerData();
|
||||
const bobParticipantId = getParticipantId(
|
||||
bobMembership.userId,
|
||||
bobMembership.deviceId,
|
||||
);
|
||||
const connectionA = {
|
||||
transport: transportA,
|
||||
} as unknown as Connection;
|
||||
const connectionB = {
|
||||
transport: transportB,
|
||||
} as unknown as Connection;
|
||||
|
||||
connectionWithPublisher.add(connectionA, []);
|
||||
connectionWithPublisher.add(connectionB, [
|
||||
mockRemoteParticipant({ identity: bobParticipantId }),
|
||||
]);
|
||||
const matrixLivekitMember$ = createMatrixLivekitMembers$({
|
||||
scope: testScope,
|
||||
membershipsWithTransport$: behavior("a", {
|
||||
a: [
|
||||
{
|
||||
membership: bobMembership,
|
||||
transport: transportA,
|
||||
},
|
||||
],
|
||||
}),
|
||||
connectionManager: aConnectionManager(
|
||||
connectionWithPublisher,
|
||||
behavior,
|
||||
),
|
||||
matrixRoom: mockMatrixRoom,
|
||||
});
|
||||
|
||||
expectObservable(matrixLivekitMember$).toBe("a", {
|
||||
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
|
||||
expect(data.length).toEqual(1);
|
||||
expect(data[0].participant).not.toBeDefined();
|
||||
expect(data[0].connection).toBeDefined();
|
||||
expect(data[0].membership).toEqual(bobMembership);
|
||||
expect(
|
||||
areLivekitTransportsEqual(
|
||||
data[0].connection!.transport,
|
||||
transportA,
|
||||
),
|
||||
).toBe(true);
|
||||
return true;
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
// let lastMatrixLkItems: MatrixLivekitMember[] = [];
|
||||
// matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => {
|
||||
// lastMatrixLkItems = items;
|
||||
// });
|
||||
|
||||
// vi.mocked(bobMembership).getTransport = vi
|
||||
// .fn()
|
||||
// .mockReturnValue(connectionA.transport);
|
||||
|
||||
// fakeMemberships$.next([bobMembership]);
|
||||
|
||||
// const lkMap = new ConnectionManagerData();
|
||||
// lkMap.add(connectionA, []);
|
||||
// lkMap.add(connectionB, [
|
||||
// mockRemoteParticipant({ identity: bobParticipantId })
|
||||
// ]);
|
||||
|
||||
// fakeManagerData$.next(lkMap);
|
||||
|
||||
// const items = lastMatrixLkItems;
|
||||
// expect(items).toHaveLength(1);
|
||||
// const item = items[0];
|
||||
|
||||
// // Assert the expected membership
|
||||
// expect(item.membership.userId).toEqual(bobMembership.userId);
|
||||
// expect(item.membership.deviceId).toEqual(bobMembership.deviceId);
|
||||
|
||||
// expect(item.participant).not.toBeDefined();
|
||||
|
||||
// // The transport info should come from the membership transports and not only from the publishing connection
|
||||
// expect(item.connection?.transport?.livekit_service_url).toEqual(
|
||||
// bobMembership.transports[0]?.livekit_service_url
|
||||
// );
|
||||
// expect(item.connection?.transport?.livekit_alias).toEqual(
|
||||
// bobMembership.transports[0]?.livekit_alias
|
||||
// );
|
||||
});
|
||||
});
|
||||
157
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
Normal file
157
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
Normal file
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type LocalParticipant as LocalLivekitParticipant,
|
||||
type RemoteParticipant as RemoteLivekitParticipant,
|
||||
} from "livekit-client";
|
||||
import {
|
||||
type LivekitTransport,
|
||||
type CallMembership,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { combineLatest, filter, map } from "rxjs";
|
||||
// eslint-disable-next-line rxjs/no-internal
|
||||
import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
|
||||
import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type Behavior } from "../../Behavior";
|
||||
import { type IConnectionManager } from "./ConnectionManager";
|
||||
import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope";
|
||||
import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname";
|
||||
import { type Connection } from "./Connection";
|
||||
|
||||
/**
|
||||
* Represent a matrix call member and his associated livekit participation.
|
||||
* `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
|
||||
* or if it has no livekit transport at all.
|
||||
*/
|
||||
export interface MatrixLivekitMember {
|
||||
membership: CallMembership;
|
||||
displayName?: string;
|
||||
participant?: LocalLivekitParticipant | RemoteLivekitParticipant;
|
||||
connection?: Connection;
|
||||
/**
|
||||
* TODO Try to remove this! Its waaay to much information.
|
||||
* Just get the member's avatar
|
||||
* @deprecated
|
||||
*/
|
||||
member: RoomMember;
|
||||
mxcAvatarUrl?: string;
|
||||
participantId: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
scope: ObservableScope;
|
||||
membershipsWithTransport$: Behavior<
|
||||
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
|
||||
>;
|
||||
connectionManager: IConnectionManager;
|
||||
// TODO this is too much information for that class,
|
||||
// apparently needed to get a room member to later get the Avatar
|
||||
// => Extract an AvatarService instead?
|
||||
// Better with just `getMember`
|
||||
matrixRoom: Pick<MatrixRoom, "getMember"> & NodeStyleEventEmitter;
|
||||
}
|
||||
// Alternative structure idea:
|
||||
// const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable<MatrixLivekitMember[]> => {
|
||||
|
||||
/**
|
||||
* Combines MatrixRTC and Livekit worlds.
|
||||
*
|
||||
* It has a small public interface:
|
||||
* - in (via constructor):
|
||||
* - an observable of CallMembership[] to track the call members (The matrix side)
|
||||
* - a `ConnectionManager` for the lk rooms (The livekit side)
|
||||
* - out (via public Observable):
|
||||
* - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data.
|
||||
*/
|
||||
export function createMatrixLivekitMembers$({
|
||||
scope,
|
||||
membershipsWithTransport$,
|
||||
connectionManager,
|
||||
matrixRoom,
|
||||
}: Props): Behavior<Epoch<MatrixLivekitMember[]>> {
|
||||
/**
|
||||
* Stream of all the call members and their associated livekit data (if available).
|
||||
*/
|
||||
|
||||
const displaynameMap$ = memberDisplaynames$(
|
||||
scope,
|
||||
matrixRoom,
|
||||
membershipsWithTransport$.pipe(mapEpoch((v) => v.map((v) => v.membership))),
|
||||
);
|
||||
|
||||
return scope.behavior(
|
||||
combineLatest([
|
||||
membershipsWithTransport$,
|
||||
connectionManager.connectionManagerData$,
|
||||
displaynameMap$,
|
||||
]).pipe(
|
||||
filter((values) =>
|
||||
values.every((value) => value.epoch === values[0].epoch),
|
||||
),
|
||||
map(
|
||||
([
|
||||
{ value: membershipsWithTransports, epoch },
|
||||
{ value: managerData },
|
||||
{ value: displaynames },
|
||||
]) => {
|
||||
const items: MatrixLivekitMember[] = membershipsWithTransports.map(
|
||||
({ membership, transport }) => {
|
||||
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
|
||||
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
|
||||
|
||||
const participants = transport
|
||||
? managerData.getParticipantForTransport(transport)
|
||||
: [];
|
||||
const participant = participants.find(
|
||||
(p) => p.identity == participantId,
|
||||
);
|
||||
const member = getRoomMemberFromRtcMember(
|
||||
membership,
|
||||
matrixRoom,
|
||||
)?.member;
|
||||
const connection = transport
|
||||
? managerData.getConnectionForTransport(transport)
|
||||
: undefined;
|
||||
const displayName = displaynames.get(participantId);
|
||||
return {
|
||||
participant,
|
||||
membership,
|
||||
connection,
|
||||
// This makes sense to add to the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar)
|
||||
// TODO Ugh this is hidign that it might be undefined!! best we remove the member entirely.
|
||||
member: member as RoomMember,
|
||||
displayName,
|
||||
mxcAvatarUrl: member?.getMxcAvatarUrl(),
|
||||
participantId,
|
||||
};
|
||||
},
|
||||
);
|
||||
return new Epoch(items, epoch);
|
||||
},
|
||||
),
|
||||
),
|
||||
// new Epoch([]),
|
||||
);
|
||||
}
|
||||
|
||||
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)
|
||||
|
||||
// TODO add this to the JS-SDK
|
||||
export function areLivekitTransportsEqual(
|
||||
t1: LivekitTransport,
|
||||
t2: LivekitTransport,
|
||||
): boolean {
|
||||
return (
|
||||
t1.livekit_service_url === t2.livekit_service_url &&
|
||||
// In case we have different lk rooms in the same SFU (depends on the livekit authorization service)
|
||||
// It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation)
|
||||
t1.livekit_alias === t2.livekit_alias
|
||||
);
|
||||
}
|
||||
299
src/state/CallViewModel/remoteMembers/displayname.test.ts
Normal file
299
src/state/CallViewModel/remoteMembers/displayname.test.ts
Normal file
@@ -0,0 +1,299 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, test, vi } from "vitest";
|
||||
import {
|
||||
type MatrixEvent,
|
||||
type RoomMember,
|
||||
type RoomState,
|
||||
RoomStateEvent,
|
||||
} from "matrix-js-sdk";
|
||||
import EventEmitter from "events";
|
||||
|
||||
import { ObservableScope } from "../../ObservableScope.ts";
|
||||
import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
|
||||
import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts";
|
||||
import { memberDisplaynames$ } from "./displayname.ts";
|
||||
|
||||
let testScope: ObservableScope;
|
||||
let mockMatrixRoom: MatrixRoom;
|
||||
|
||||
/*
|
||||
* To be populated in the test setup.
|
||||
* Maps userId to a partial/mock RoomMember object.
|
||||
*/
|
||||
let fakeMembersMap: Map<string, Partial<RoomMember>>;
|
||||
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
fakeMembersMap = new Map<string, Partial<RoomMember>>();
|
||||
|
||||
const roomEmitter = new EventEmitter();
|
||||
mockMatrixRoom = {
|
||||
on: roomEmitter.on.bind(roomEmitter),
|
||||
off: roomEmitter.off.bind(roomEmitter),
|
||||
emit: roomEmitter.emit.bind(roomEmitter),
|
||||
// addListener: roomEmitter.addListener.bind(roomEmitter),
|
||||
// removeListener: roomEmitter.removeListener.bind(roomEmitter),
|
||||
getMember: vi.fn().mockImplementation((userId: string) => {
|
||||
const member = fakeMembersMap.get(userId);
|
||||
if (member) {
|
||||
return member as RoomMember;
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
} as unknown as MatrixRoom;
|
||||
});
|
||||
|
||||
function fakeMemberWith(data: Partial<RoomMember>): void {
|
||||
const userId = data.userId || "@alice:example.com";
|
||||
const member: Partial<RoomMember> = {
|
||||
userId: userId,
|
||||
rawDisplayName: data.rawDisplayName ?? userId,
|
||||
...data,
|
||||
} as unknown as RoomMember;
|
||||
fakeMembersMap.set(userId, member);
|
||||
// return member as RoomMember;
|
||||
}
|
||||
|
||||
function updateDisplayName(
|
||||
userId: `@${string}:${string}`,
|
||||
newDisplayName: string,
|
||||
): void {
|
||||
const member = fakeMembersMap.get(userId);
|
||||
if (member) {
|
||||
member.rawDisplayName = newDisplayName;
|
||||
// Emit the event to notify listeners
|
||||
mockMatrixRoom.emit(
|
||||
RoomStateEvent.Members,
|
||||
{} as unknown as MatrixEvent,
|
||||
{} as unknown as RoomState,
|
||||
member as RoomMember,
|
||||
);
|
||||
} else {
|
||||
throw new Error(`No member found with userId: ${userId}`);
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
fakeMembersMap.clear();
|
||||
});
|
||||
|
||||
test("should always have our own user", () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("a", {
|
||||
a: [],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
expectObservable(dn$).toBe("a", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "@local:example.com"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function setUpBasicRoom(): void {
|
||||
fakeMemberWith({ userId: "@local:example.com", rawDisplayName: "it's a me" });
|
||||
fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" });
|
||||
fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" });
|
||||
fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" });
|
||||
fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" });
|
||||
fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" });
|
||||
fakeMemberWith({ userId: "@no-name:foo.bar" });
|
||||
}
|
||||
|
||||
test("should get displayName for users", () => {
|
||||
setUpBasicRoom();
|
||||
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("a", {
|
||||
a: [
|
||||
mockCallMembership("@alice:example.com", "DEVICE1"),
|
||||
mockCallMembership("@bob:example.com", "DEVICE1"),
|
||||
],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
expectObservable(dn$).toBe("a", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@alice:example.com:DEVICE1", "Alice"],
|
||||
["@bob:example.com:DEVICE1", "Bob"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("should use userId if no display name", () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
setUpBasicRoom();
|
||||
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("a", {
|
||||
a: [mockCallMembership("@no-name:foo.bar", "D000")],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
expectObservable(dn$).toBe("a", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@no-name:foo.bar:D000", "@no-name:foo.bar"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("should disambiguate users with same display name", () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
setUpBasicRoom();
|
||||
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("a", {
|
||||
a: [
|
||||
mockCallMembership("@bob:example.com", "DEVICE1"),
|
||||
mockCallMembership("@bob:example.com", "DEVICE2"),
|
||||
mockCallMembership("@bob:foo.bar", "BOB000"),
|
||||
mockCallMembership("@carl:example.com", "C000"),
|
||||
mockCallMembership("@evil:example.com", "E000"),
|
||||
],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
expectObservable(dn$).toBe("a", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
|
||||
["@bob:example.com:DEVICE2", "Bob (@bob:example.com)"],
|
||||
["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"],
|
||||
["@carl:example.com:C000", "Carl (@carl:example.com)"],
|
||||
["@evil:example.com:E000", "Carl (@evil:example.com)"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("should disambiguate when needed", () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
setUpBasicRoom();
|
||||
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("ab", {
|
||||
a: [mockCallMembership("@bob:example.com", "DEVICE1")],
|
||||
b: [
|
||||
mockCallMembership("@bob:example.com", "DEVICE1"),
|
||||
mockCallMembership("@bob:foo.bar", "BOB000"),
|
||||
],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
expectObservable(dn$).toBe("ab", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:DEVICE1", "Bob"],
|
||||
]),
|
||||
b: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
|
||||
["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test.skip("should keep disambiguated name when other leave", () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
setUpBasicRoom();
|
||||
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("ab", {
|
||||
a: [
|
||||
mockCallMembership("@bob:example.com", "DEVICE1"),
|
||||
mockCallMembership("@bob:foo.bar", "BOB000"),
|
||||
],
|
||||
b: [mockCallMembership("@bob:example.com", "DEVICE1")],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
expectObservable(dn$).toBe("ab", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
|
||||
["@bob:foo.bar:BOB000", "Bob (@bob:foo.bar)"],
|
||||
]),
|
||||
b: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:DEVICE1", "Bob (@bob:example.com)"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("should disambiguate on name change", () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable }) => {
|
||||
setUpBasicRoom();
|
||||
|
||||
const dn$ = memberDisplaynames$(
|
||||
testScope,
|
||||
mockMatrixRoom,
|
||||
cold("a", {
|
||||
a: [
|
||||
mockCallMembership("@bob:example.com", "B000"),
|
||||
mockCallMembership("@carl:example.com", "C000"),
|
||||
],
|
||||
}),
|
||||
"@local:example.com",
|
||||
"DEVICE000",
|
||||
);
|
||||
|
||||
schedule("-a", {
|
||||
a: () => {
|
||||
updateDisplayName("@carl:example.com", "Bob");
|
||||
},
|
||||
});
|
||||
|
||||
expectObservable(dn$).toBe("ab", {
|
||||
a: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:B000", "Bob"],
|
||||
["@carl:example.com:C000", "Carl"],
|
||||
]),
|
||||
b: new Map<string, string>([
|
||||
["@local:example.com:DEVICE000", "it's a me"],
|
||||
["@bob:example.com:B000", "Bob (@bob:example.com)"],
|
||||
["@carl:example.com:C000", "Bob (@carl:example.com)"],
|
||||
]),
|
||||
});
|
||||
});
|
||||
});
|
||||
87
src/state/CallViewModel/remoteMembers/displayname.ts
Normal file
87
src/state/CallViewModel/remoteMembers/displayname.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
Copyright 2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
|
||||
import {
|
||||
combineLatest,
|
||||
fromEvent,
|
||||
map,
|
||||
type Observable,
|
||||
startWith,
|
||||
} from "rxjs";
|
||||
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
|
||||
// eslint-disable-next-line rxjs/no-internal
|
||||
import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
|
||||
|
||||
import { Epoch, type ObservableScope } from "../../ObservableScope";
|
||||
import {
|
||||
calculateDisplayName,
|
||||
shouldDisambiguate,
|
||||
} from "../../../utils/displayname";
|
||||
import { type Behavior } from "../../Behavior";
|
||||
|
||||
/**
|
||||
* Displayname for each member of the call. This will disambiguate
|
||||
* any displayname that clashes with another member. Only members
|
||||
* joined to the call are considered here.
|
||||
*
|
||||
* @returns Map<member.id, displayname> uses the rtc member idenitfier as the key.
|
||||
*/
|
||||
// don't do this work more times than we need to. This is achieved by converting to a behavior:
|
||||
export const memberDisplaynames$ = (
|
||||
scope: ObservableScope,
|
||||
matrixRoom: Pick<MatrixRoom, "getMember"> & NodeStyleEventEmitter,
|
||||
memberships$: Observable<Epoch<CallMembership[]>>,
|
||||
): Behavior<Epoch<Map<string, string>>> =>
|
||||
scope.behavior(
|
||||
combineLatest([
|
||||
// Handle call membership changes
|
||||
memberships$,
|
||||
// Additionally handle display name changes (implicitly reacting to them)
|
||||
fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)),
|
||||
// TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
|
||||
]).pipe(
|
||||
map(([epochMemberships, _displayNames]) => {
|
||||
const { epoch, value: memberships } = epochMemberships;
|
||||
const displaynameMap = new Map<string, string>();
|
||||
const room = matrixRoom;
|
||||
|
||||
// We only consider RTC members for disambiguation as they are the only visible members.
|
||||
for (const rtcMember of memberships) {
|
||||
// TODO a hard-coded participant ID ? should use rtcMember.membershipID instead?
|
||||
const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`;
|
||||
const { member } = getRoomMemberFromRtcMember(rtcMember, room);
|
||||
if (!member) {
|
||||
logger.error(
|
||||
"Could not find member for participant id:",
|
||||
matrixIdentifier,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const disambiguate = shouldDisambiguate(member, memberships, room);
|
||||
displaynameMap.set(
|
||||
matrixIdentifier,
|
||||
calculateDisplayName(member, disambiguate),
|
||||
);
|
||||
}
|
||||
return new Epoch(displaynameMap, epoch);
|
||||
}),
|
||||
),
|
||||
new Epoch(new Map<string, string>()),
|
||||
);
|
||||
|
||||
export function getRoomMemberFromRtcMember(
|
||||
rtcMember: CallMembership,
|
||||
room: Pick<MatrixRoom, "getMember">,
|
||||
): { id: string; member: RoomMember | undefined } {
|
||||
return {
|
||||
id: rtcMember.userId + ":" + rtcMember.deviceId,
|
||||
member: room.getMember(rtcMember.userId) ?? undefined,
|
||||
};
|
||||
}
|
||||
220
src/state/CallViewModel/remoteMembers/integration.test.ts
Normal file
220
src/state/CallViewModel/remoteMembers/integration.test.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { test, vi, expect, beforeEach, afterEach } from "vitest";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { type Room as LivekitRoom } from "livekit-client";
|
||||
import EventEmitter from "events";
|
||||
import fetchMock from "fetch-mock";
|
||||
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
type Epoch,
|
||||
ObservableScope,
|
||||
trackEpoch,
|
||||
} from "../../ObservableScope.ts";
|
||||
import { ECConnectionFactory } from "./ConnectionFactory.ts";
|
||||
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
|
||||
import {
|
||||
mockCallMembership,
|
||||
mockMediaDevices,
|
||||
withTestScheduler,
|
||||
} from "../../../utils/test.ts";
|
||||
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import {
|
||||
areLivekitTransportsEqual,
|
||||
createMatrixLivekitMembers$,
|
||||
type MatrixLivekitMember,
|
||||
} from "./MatrixLivekitMembers.ts";
|
||||
import { createConnectionManager$ } from "./ConnectionManager.ts";
|
||||
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
|
||||
|
||||
// Test the integration of ConnectionManager and MatrixLivekitMerger
|
||||
|
||||
let testScope: ObservableScope;
|
||||
let ecConnectionFactory: ECConnectionFactory;
|
||||
let mockClient: OpenIDClientParts;
|
||||
let lkRoomFactory: () => LivekitRoom;
|
||||
let mockMatrixRoom: MatrixRoom;
|
||||
|
||||
const createdMockLivekitRooms: Map<string, LivekitRoom> = new Map();
|
||||
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
mockClient = {
|
||||
getOpenIdToken: vi.fn().mockReturnValue(""),
|
||||
getDeviceId: vi.fn().mockReturnValue("DEV000"),
|
||||
};
|
||||
|
||||
lkRoomFactory = vi.fn().mockImplementation(() => {
|
||||
const emitter = new EventEmitter();
|
||||
const base = {
|
||||
on: emitter.on.bind(emitter),
|
||||
off: emitter.off.bind(emitter),
|
||||
emit: emitter.emit.bind(emitter),
|
||||
disconnect: vi.fn(),
|
||||
remoteParticipants: new Map(),
|
||||
} as unknown as LivekitRoom;
|
||||
|
||||
vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => {
|
||||
createdMockLivekitRooms.set(url, base);
|
||||
});
|
||||
return base;
|
||||
});
|
||||
|
||||
ecConnectionFactory = new ECConnectionFactory(
|
||||
mockClient,
|
||||
mockMediaDevices({}),
|
||||
new BehaviorSubject<ProcessorState>({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
undefined,
|
||||
false,
|
||||
lkRoomFactory,
|
||||
);
|
||||
|
||||
//TODO a bit annoying to have to do a http mock?
|
||||
fetchMock.post(`path:/sfu/get`, (url) => {
|
||||
const domain = new URL(url).hostname; // Extract the domain from the URL
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
url: `wss://${domain}/livekit/sfu`,
|
||||
jwt: "ATOKEN",
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
mockMatrixRoom = vi.mocked<MatrixRoom>({
|
||||
getMember: vi.fn().mockImplementation((userId: string) => {
|
||||
return {
|
||||
userId,
|
||||
rawDisplayName: userId.replace("@", "").replace(":example.org", ""),
|
||||
getMxcAvatarUrl: vi.fn().mockReturnValue(null),
|
||||
} as unknown as RoomMember;
|
||||
}),
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
} as unknown as MatrixRoom);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testScope.end();
|
||||
fetchMock.reset();
|
||||
});
|
||||
|
||||
test("bob, carl, then bob joining no tracks yet", () => {
|
||||
withTestScheduler(({ expectObservable, behavior, scope }) => {
|
||||
const bobMembership = mockCallMembership("@bob:example.com", "BDEV000");
|
||||
const carlMembership = mockCallMembership("@carl:example.com", "CDEV000");
|
||||
const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000");
|
||||
|
||||
const eMarble = "abc";
|
||||
const vMarble = "abc";
|
||||
const memberships$ = scope.behavior(
|
||||
behavior(eMarble, {
|
||||
a: [bobMembership],
|
||||
b: [bobMembership, carlMembership],
|
||||
c: [bobMembership, carlMembership, daveMembership],
|
||||
}).pipe(trackEpoch()),
|
||||
);
|
||||
|
||||
const membershipsAndTransports = membershipsAndTransports$(
|
||||
testScope,
|
||||
memberships$,
|
||||
);
|
||||
|
||||
const connectionManager = createConnectionManager$({
|
||||
scope: testScope,
|
||||
connectionFactory: ecConnectionFactory,
|
||||
inputTransports$: membershipsAndTransports.transports$,
|
||||
});
|
||||
|
||||
const matrixLivekitItems$ = createMatrixLivekitMembers$({
|
||||
scope: testScope,
|
||||
membershipsWithTransport$:
|
||||
membershipsAndTransports.membershipsWithTransport$,
|
||||
connectionManager,
|
||||
matrixRoom: mockMatrixRoom,
|
||||
});
|
||||
|
||||
expectObservable(matrixLivekitItems$).toBe(vMarble, {
|
||||
a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
|
||||
const items = e.value;
|
||||
expect(items.length).toBe(1);
|
||||
const item = items[0]!;
|
||||
expect(item.membership).toStrictEqual(bobMembership);
|
||||
expect(
|
||||
areLivekitTransportsEqual(
|
||||
item.connection!.transport,
|
||||
bobMembership.transports[0]! as LivekitTransport,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(item.participant).toBeUndefined();
|
||||
return true;
|
||||
}),
|
||||
b: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
|
||||
const items = e.value;
|
||||
expect(items.length).toBe(2);
|
||||
|
||||
{
|
||||
const item = items[0]!;
|
||||
expect(item.membership).toStrictEqual(bobMembership);
|
||||
expect(item.participant).toBeUndefined();
|
||||
}
|
||||
|
||||
{
|
||||
const item = items[1]!;
|
||||
expect(item.membership).toStrictEqual(carlMembership);
|
||||
expect(item.participantId).toStrictEqual(
|
||||
`${carlMembership.userId}:${carlMembership.deviceId}`,
|
||||
);
|
||||
expect(
|
||||
areLivekitTransportsEqual(
|
||||
item.connection!.transport,
|
||||
carlMembership.transports[0]! as LivekitTransport,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(item.participant).toBeUndefined();
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
c: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
|
||||
const items = e.value;
|
||||
logger.info(`E Items length: ${items.length}`);
|
||||
expect(items.length).toBe(3);
|
||||
{
|
||||
expect(items[0]!.membership).toStrictEqual(bobMembership);
|
||||
}
|
||||
|
||||
{
|
||||
expect(items[1]!.membership).toStrictEqual(carlMembership);
|
||||
}
|
||||
|
||||
{
|
||||
const item = items[2]!;
|
||||
expect(item.membership).toStrictEqual(daveMembership);
|
||||
expect(item.participantId).toStrictEqual(
|
||||
`${daveMembership.userId}:${daveMembership.deviceId}`,
|
||||
);
|
||||
expect(
|
||||
areLivekitTransportsEqual(
|
||||
item.connection!.transport,
|
||||
daveMembership.transports[0]! as LivekitTransport,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(item.participant).toBeUndefined();
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
x: expect.anything(),
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user