studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

start moving over/removing things from the CallViewModel

Browse Source
This commit is contained in:
Timo K
2025-10-30 01:13:06 +01:00
parent 6b513534f1
commit c8ef8d6a24
7 changed files with 231 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files

418
src/state/CallViewModel.ts
View File

@@ -137,7 +137,16 @@ import {
import { ElementCallError, UnknownCallError } from "../utils/errors.ts"; import { ElementCallError, UnknownCallError } from "../utils/errors.ts";
import { ObservableScope } from "./ObservableScope.ts"; import { ObservableScope } from "./ObservableScope.ts";
import { memberDisplaynames$ } from "./remoteMembers/displayname.ts"; import { memberDisplaynames$ } from "./remoteMembers/displayname.ts";
import { ConnectionManager } from "./remoteMembers/ConnectionManager.ts";
import { MatrixLivekitMerger } from "./remoteMembers/matrixLivekitMerger.ts";
//TODO
// Larger rename
// member,membership -> rtcMember
// participant -> livekitParticipant
// matrixLivekitItem -> callMember
// js-sdk
// callMembership -> rtcMembership
export interface CallViewModelOptions { export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem; encryptionSystem: EncryptionSystem;
autoLeaveWhenOthersLeft?: boolean; autoLeaveWhenOthersLeft?: boolean;
@@ -205,6 +214,29 @@ export class CallViewModel {
null, null,
); );
private memberships$ = this.scope.behavior(
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
(_, memberships: CallMembership[]) => memberships,
),
);
private connectionManager = new ConnectionManager(
this.scope,
this.matrixRoom.client,
this.mediaDevices,
this.trackProcessorState$,
this.e2eeLivekitOptions(),
);
private matrixLivekitMerger = new MatrixLivekitMerger(
this.scope,
this.memberships$,
this.connectionManager,
this.matrixRoom,
);
/** /**
* If there is a configuration error with the call (e.g. misconfigured E2EE). * If there is a configuration error with the call (e.g. misconfigured E2EE).
* This is a fatal error that prevents the call from being created/joined. * This is a fatal error that prevents the call from being created/joined.
@@ -221,7 +253,7 @@ export class CallViewModel {
this.join$.next(); this.join$.next();
} }
// CODESMALL // CODESMELL?
// This is functionally the same Observable as leave$, except here it's // This is functionally the same Observable as leave$, except here it's
// hoisted to the top of the class. This enables the cyclic dependency between // hoisted to the top of the class. This enables the cyclic dependency between
// leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ ->
@@ -302,112 +334,28 @@ export class CallViewModel {
), ),
); );
// DISCUSSION move to ConnectionManager // // DISCUSSION move to ConnectionManager
/** // public readonly livekitConnectionState$ =
* The local connection over which we will publish our media. It could // // TODO: This options.connectionState$ behavior is a small hack inserted
* possibly also have some remote users' media available on it. // // here to facilitate testing. This would likely be better served by
* null when not joined. // // breaking CallViewModel down into more naturally testable components.
*/ // this.options.connectionState$ ??
private readonly localConnection$: Behavior<Async<PublishConnection> | null> = // this.scope.behavior<ConnectionState>(
this.scope.behavior( // this.localConnection$.pipe(
generateKeyed$< // switchMap((c) =>
Async<LivekitTransport> | null, // c?.state === "ready"
PublishConnection, // ? // TODO mapping to ConnectionState for compatibility, but we should use the full state?
Async<PublishConnection> | null // c.value.state$.pipe(
>( // switchMap((s) => {
this.localTransport$, // if (s.state === "ConnectedToLkRoom")
(transport, createOrGet) => // return s.connectionState$;
transport && // return of(ConnectionState.Disconnected);
mapAsync(transport, (transport) => // }),
createOrGet( // )
// Stable key that uniquely idenifies the transport // : of(ConnectionState.Disconnected),
JSON.stringify({ // ),
url: transport.livekit_service_url, // ),
alias: transport.livekit_alias, // );
}),
(scope) =>
new PublishConnection(
{
transport,
client: this.matrixRoom.client,
scope,
remoteTransports$: this.remoteTransports$,
livekitRoomFactory: this.options.livekitRoomFactory,
},
this.mediaDevices,
this.muteStates,
this.e2eeLivekitOptions(),
this.scope.behavior(this.trackProcessorState$),
),
),
),
),
);
// DISCUSSION move to ConnectionManager
public readonly livekitConnectionState$ =
// TODO: This options.connectionState$ behavior is a small hack inserted
// here to facilitate testing. This would likely be better served by
// breaking CallViewModel down into more naturally testable components.
this.options.connectionState$ ??
this.scope.behavior<ConnectionState>(
this.localConnection$.pipe(
switchMap((c) =>
c?.state === "ready"
? // TODO mapping to ConnectionState for compatibility, but we should use the full state?
c.value.state$.pipe(
switchMap((s) => {
if (s.state === "ConnectedToLkRoom")
return s.connectionState$;
return of(ConnectionState.Disconnected);
}),
)
: of(ConnectionState.Disconnected),
),
),
);
/**
* A list of the connections that should be active at any given time.
*/
// DISCUSSION move to ConnectionManager
private readonly connections$ = this.scope.behavior<Connection[]>(
combineLatest(
[this.localConnection$, this.remoteConnections$],
(local, remote) => [
...(local?.state === "ready" ? [local.value] : []),
...remote.values(),
],
),
);
/**
* Emits with connections whenever they should be started or stopped.
*/
// DISCUSSION move to ConnectionManager
private readonly connectionInstructions$ = this.connections$.pipe(
pairwise(),
map(([prev, next]) => {
const start = new Set(next.values());
for (const connection of prev) start.delete(connection);
const stop = new Set(prev.values());
for (const connection of next) stop.delete(connection);
return { start, stop };
}),
);
public readonly allLivekitRooms$ = this.scope.behavior(
this.connections$.pipe(
map((connections) =>
[...connections.values()].map((c) => ({
room: c.livekitRoom,
url: c.transport.livekit_service_url,
isLocal: c instanceof PublishConnection,
})),
),
),
);
private readonly userId = this.matrixRoom.client.getUserId()!; private readonly userId = this.matrixRoom.client.getUserId()!;
private readonly deviceId = this.matrixRoom.client.getDeviceId()!; private readonly deviceId = this.matrixRoom.client.getDeviceId()!;
@@ -450,114 +398,6 @@ export class CallViewModel {
), ),
); );
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
*/
// DISCUSSION own membership manager
private readonly connected$ = this.scope.behavior(
and$(
this.matrixConnected$,
this.livekitConnectionState$.pipe(
map((state) => state === ConnectionState.Connected),
),
),
);
/**
* Whether we should tell the user that we're reconnecting to the call.
*/
// DISCUSSION own membership manager
public readonly reconnecting$ = this.scope.behavior(
this.connected$.pipe(
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
{ connectedPreviously: false, reconnecting: false },
),
map(({ reconnecting }) => reconnecting),
),
);
/**
* Lists the transports used by ourselves, plus all other MatrixRTC session
* members. For completeness this also lists the preferred transport and
* whether we are in multi-SFU mode or sticky events mode (because
* advertisedTransport$ wants to read them at the same time, and bundling data
* together when it might change together is what you have to do in RxJS to
* avoid reading inconsistent state or observing too many changes.)
*/
// TODO-MULTI-SFU find a better name for this. With the addition of sticky events it's no longer just about transports.
// DISCUSS move the local part to the own membership file
private readonly transports$: Behavior<{
local: Async<LivekitTransport>;
remote: { membership: CallMembership; transport: LivekitTransport }[];
preferred: Async<LivekitTransport>;
multiSfu: boolean;
preferStickyEvents: boolean;
} | null> = this.scope.behavior(
this.joined$.pipe(
switchMap((joined) =>
joined
? combineLatest(
[
this.preferredTransport$,
this.memberships$,
multiSfu.value$,
preferStickyEvents.value$,
],
(preferred, memberships, preferMultiSfu, preferStickyEvents) => {
// Multi-SFU must be implicitly enabled when using sticky events
const multiSfu = preferStickyEvents || preferMultiSfu;
const oldestMembership =
this.matrixRTCSession.getOldestMembership();
const remote = memberships.flatMap((m) => {
if (m.userId === this.userId && m.deviceId === this.deviceId)
return [];
const t = m.getTransport(oldestMembership ?? m);
return t && isLivekitTransport(t)
? [{ membership: m, transport: t }]
: [];
});
let local = preferred;
if (!multiSfu) {
const oldest = this.matrixRTCSession.getOldestMembership();
if (oldest !== undefined) {
const selection = oldest.getTransport(oldest);
// TODO selection can be null if no transport is configured should we report an error?
if (selection && isLivekitTransport(selection))
local = ready(selection);
}
}
if (local.state === "error") {
this._configError$.next(
local.value instanceof ElementCallError
? local.value
: new UnknownCallError(local.value),
);
}
return {
local,
remote,
preferred,
multiSfu,
preferStickyEvents,
};
},
)
: of(null),
),
),
);
/** /**
* Whether various media/event sources should pretend to be disconnected from * Whether various media/event sources should pretend to be disconnected from
* all network input, even if their connection still technically works. * all network input, even if their connection still technically works.
@@ -569,95 +409,7 @@ export class CallViewModel {
// DISCUSSION own membership manager ALSO this probably can be simplifis // DISCUSSION own membership manager ALSO this probably can be simplifis
private readonly pretendToBeDisconnected$ = this.reconnecting$; private readonly pretendToBeDisconnected$ = this.reconnecting$;
/** public readonly audioParticipants$; // now will be created based on the connectionmanager
* Lists, for each LiveKit room, the LiveKit participants whose media should
* be presented.
*/
private readonly participantsByRoom$ = this.scope.behavior<
{
livekitRoom: LivekitRoom;
url: string; // Included for use as a React key
participants: {
id: string;
participant: LocalParticipant | RemoteParticipant | undefined;
member: RoomMember;
}[];
}[]
>(
// TODO: Move this logic into Connection/PublishConnection if possible
this.localConnection$
.pipe(
switchMap((localConnection) => {
if (localConnection?.state !== "ready") return [];
const memberError = (): never => {
throw new Error("No room member for call membership");
};
const localParticipant = {
id: `${this.userId}:${this.deviceId}`,
participant: localConnection.value.livekitRoom.localParticipant,
member:
this.matrixRoom.getMember(this.userId ?? "") ?? memberError(),
};
return this.remoteConnections$.pipe(
switchMap((remoteConnections) =>
combineLatest(
[localConnection.value, ...remoteConnections].map((c) =>
c.publishingParticipants$.pipe(
map((ps) => {
const participants: {
id: string;
participant:
| LocalParticipant
| RemoteParticipant
| undefined;
member: RoomMember;
}[] = ps.map(({ participant, membership }) => ({
id: `${membership.userId}:${membership.deviceId}`,
participant,
member:
getRoomMemberFromRtcMember(
membership,
this.matrixRoom,
)?.member ?? memberError(),
}));
if (c === localConnection.value)
participants.push(localParticipant);
return {
livekitRoom: c.livekitRoom,
url: c.transport.livekit_service_url,
participants,
};
}),
),
),
),
),
);
}),
)
.pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)),
);
/**
* Lists, for each LiveKit room, the LiveKit participants whose audio should
* be rendered.
*/
// (This is effectively just participantsByRoom$ with a stricter type)
public readonly audioParticipants$ = this.scope.behavior(
this.participantsByRoom$.pipe(
map((data) =>
data.map(({ livekitRoom, url, participants }) => ({
livekitRoom,
url,
participants: participants.flatMap(({ participant }) =>
participant instanceof RemoteParticipant ? [participant] : [],
),
})),
),
),
);
public readonly handsRaised$ = this.scope.behavior( public readonly handsRaised$ = this.scope.behavior(
this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)),
@@ -677,17 +429,19 @@ export class CallViewModel {
), ),
); );
memberDisplaynames$ = memberDisplaynames$( // Now will be added to the matricLivekitMerger
this.matrixRoom, // memberDisplaynames$ = memberDisplaynames$(
this.memberships$, // this.matrixRoom,
this.scope, // this.memberships$,
this.userId, // this.scope,
this.deviceId, // this.userId,
); // this.deviceId,
// );
/** /**
* List of MediaItems that we want to have tiles for. * List of MediaItems that we want to have tiles for.
*/ */
// TODO KEEP THIS!! and adapt it to what our membershipManger returns
private readonly mediaItems$ = this.scope.behavior<MediaItem[]>( private readonly mediaItems$ = this.scope.behavior<MediaItem[]>(
generateKeyed$< generateKeyed$<
[typeof this.participantsByRoom$.value, number], [typeof this.participantsByRoom$.value, number],
@@ -790,10 +544,12 @@ export class CallViewModel {
* - There can be multiple participants for one Matrix user if they join from * - There can be multiple participants for one Matrix user if they join from
* multiple devices. * multiple devices.
*/ */
// TODO KEEP THIS!! and adapt it to what our membershipManger returns
public readonly participantCount$ = this.scope.behavior( public readonly participantCount$ = this.scope.behavior(
this.memberships$.pipe(map((ms) => ms.length)), this.memberships$.pipe(map((ms) => ms.length)),
); );
// TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$}
private readonly allOthersLeft$ = this.memberships$.pipe( private readonly allOthersLeft$ = this.memberships$.pipe(
pairwise(), pairwise(),
filter( filter(
@@ -1687,46 +1443,8 @@ export class CallViewModel {
private readonly reactionsSubject$: Observable< private readonly reactionsSubject$: Observable<
Record<string, ReactionInfo> Record<string, ReactionInfo>
>, >,
private readonly trackProcessorState$: Observable<ProcessorState>, private readonly trackProcessorState$: Behavior<ProcessorState>,
) { ) {
// Start and stop local and remote connections as needed
// DISCUSSION connection manager
this.connectionInstructions$
.pipe(this.scope.bind())
.subscribe(({ start, stop }) => {
for (const c of stop) {
logger.info(`Disconnecting from ${c.transport.livekit_service_url}`);
c.stop().catch((err) => {
// TODO: better error handling
logger.error(
`Fail to stop connection to ${c.transport.livekit_service_url}`,
err,
);
});
}
for (const c of start) {
c.start().then(
() =>
logger.info(`Connected to ${c.transport.livekit_service_url}`),
(e) => {
// We only want to report fatal errors `_configError$` for the publish connection.
// If there is an error with another connection, it will not terminate the call and will be displayed
// on eacn tile.
if (
c instanceof PublishConnection &&
e instanceof ElementCallError
) {
this._configError$.next(e);
}
logger.error(
`Failed to start connection to ${c.transport.livekit_service_url}`,
e,
);
},
);
}
});
// Start and stop session membership as needed // Start and stop session membership as needed
this.scope.reconcile(this.advertisedTransport$, async (advertised) => { this.scope.reconcile(this.advertisedTransport$, async (advertised) => {
if (advertised !== null) { if (advertised !== null) {

160
src/state/ownMember/OwnMembership.ts
View File

@@ -19,84 +19,116 @@ const ownMembership$ = (
connected: Behavior<boolean>; connected: Behavior<boolean>;
transport: Behavior<LivekitTransport | null>; transport: Behavior<LivekitTransport | null>;
} => { } => {
const userId = this.matrixRoom.client.getUserId()!;
const deviceId = this.matrixRoom.client.getDeviceId()!;
const connection = connectionManager.registerTransports( const connection = connectionManager.registerTransports(
constant([transport]), constant([transport]),
); );
const publisher = new Publisher(connection); const publisher = new Publisher(connection);
// HOW IT WAS PREVIEOUSLY CREATED
// new PublishConnection(
// {
// transport,
// client: this.matrixRoom.client,
// scope,
// remoteTransports$: this.remoteTransports$,
// livekitRoomFactory: this.options.livekitRoomFactory,
// },
// this.mediaDevices,
// this.muteStates,
// this.e2eeLivekitOptions(),
// this.scope.behavior(this.trackProcessorState$),
// ),
/** /**
* Lists the transports used by ourselves, plus all other MatrixRTC session * The transport that we would personally prefer to publish on (if not for the
* members. For completeness this also lists the preferred transport and * transport preferences of others, perhaps).
* whether we are in multi-SFU mode or sticky events mode (because
* advertisedTransport$ wants to read them at the same time, and bundling data
* together when it might change together is what you have to do in RxJS to
* avoid reading inconsistent state or observing too many changes.)
*/ */
// TODO-MULTI-SFU find a better name for this. With the addition of sticky events it's no longer just about transports. // DISCUSS move to ownMembership
// DISCUSS move to MatrixLivekitMerger private readonly preferredTransport$ = this.scope.behavior(
const transport$: Behavior<{ async$(makeTransport(this.matrixRTCSession)),
local: Async<LivekitTransport>; );
preferred: Async<LivekitTransport>;
/**
* The transport over which we should be actively publishing our media.
* null when not joined.
*/
// DISCUSSION ownMembershipManager
private readonly localTransport$: Behavior<Async<LivekitTransport> | null> =
this.scope.behavior(
this.transports$.pipe(
map((transports) => transports?.local ?? null),
distinctUntilChanged<Async<LivekitTransport> | null>(deepCompare),
),
);
/**
* The transport we should advertise in our MatrixRTC membership (plus whether
* it is a multi-SFU transport and whether we should use sticky events).
*/
// DISCUSSION ownMembershipManager
private readonly advertisedTransport$: Behavior<{
multiSfu: boolean; multiSfu: boolean;
preferStickyEvents: boolean; preferStickyEvents: boolean;
transport: LivekitTransport;
} | null> = this.scope.behavior( } | null> = this.scope.behavior(
this.joined$.pipe( this.transports$.pipe(
switchMap((joined) => map((transports) =>
joined transports?.local.state === "ready" &&
? combineLatest( transports.preferred.state === "ready"
[ ? {
this.preferredTransport$, multiSfu: transports.multiSfu,
this.memberships$, preferStickyEvents: transports.preferStickyEvents,
multiSfu.value$, // In non-multi-SFU mode we should always advertise the preferred
preferStickyEvents.value$, // SFU to minimize the number of membership updates
], transport: transports.multiSfu
(preferred, memberships, preferMultiSfu, preferStickyEvents) => { ? transports.local.value
// Multi-SFU must be implicitly enabled when using sticky events : transports.preferred.value,
const multiSfu = preferStickyEvents || preferMultiSfu; }
: null,
),
distinctUntilChanged<{
multiSfu: boolean;
preferStickyEvents: boolean;
transport: LivekitTransport;
} | null>(deepCompare),
),
);
const oldestMembership = // MATRIX RELATED
this.matrixRTCSession.getOldestMembership(); //
const remote = memberships.flatMap((m) => { /**
if (m.userId === this.userId && m.deviceId === this.deviceId) * Whether we are "fully" connected to the call. Accounts for both the
return []; * connection to the MatrixRTC session and the LiveKit publish connection.
const t = m.getTransport(oldestMembership ?? m); */
return t && isLivekitTransport(t) // DISCUSSION own membership manager
? [{ membership: m, transport: t }] private readonly connected$ = this.scope.behavior(
: []; and$(
}); this.matrixConnected$,
this.livekitConnectionState$.pipe(
let local = preferred; map((state) => state === ConnectionState.Connected),
if (!multiSfu) {
const oldest = this.matrixRTCSession.getOldestMembership();
if (oldest !== undefined) {
const selection = oldest.getTransport(oldest);
// TODO selection can be null if no transport is configured should we report an error?
if (selection && isLivekitTransport(selection))
local = ready(selection);
}
}
if (local.state === "error") {
this._configError$.next(
local.value instanceof ElementCallError
? local.value
: new UnknownCallError(local.value),
);
}
return {
local,
remote,
preferred,
multiSfu,
preferStickyEvents,
};
},
)
: of(null),
), ),
), ),
); );
/**
* Whether we should tell the user that we're reconnecting to the call.
*/
// DISCUSSION own membership manager
public readonly reconnecting$ = this.scope.behavior(
this.connected$.pipe(
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
{ connectedPreviously: false, reconnecting: false },
),
map(({ reconnecting }) => reconnecting),
),
);
return { connected: true, transport$ }; return { connected: true, transport$ };
}; };

5
src/state/remoteMembers/Connection.test.ts
View File

@@ -17,7 +17,7 @@ import {
} from "vitest"; } from "vitest";
import { BehaviorSubject, of } from "rxjs"; import { BehaviorSubject, of } from "rxjs";
import { import {
ConnectionState, ConnectionState as LivekitConnectionState,
type LocalParticipant, type LocalParticipant,
type RemoteParticipant, type RemoteParticipant,
type Room as LivekitRoom, type Room as LivekitRoom,
@@ -36,12 +36,11 @@ import {
type ConnectionOpts, type ConnectionOpts,
type ConnectionState, type ConnectionState,
type PublishingParticipant, type PublishingParticipant,
RemoteConnection, Connection,
} from "./Connection.ts"; } from "./Connection.ts";
import { ObservableScope } from "../ObservableScope.ts"; import { ObservableScope } from "../ObservableScope.ts";
import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts"; import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../utils/errors.ts"; import { FailToGetOpenIdToken } from "../../utils/errors.ts";
import { PublishConnection } from "../ownMember/Publisher.ts";
import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts"; import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts";
import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx"; import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
import { type MuteStates } from "../MuteStates.ts"; import { type MuteStates } from "../MuteStates.ts";

10
src/state/remoteMembers/ConnectionManager.ts
View File

@@ -42,11 +42,11 @@ export type ParticipantByMemberIdMap = Map<
export class ConnectionManager { export class ConnectionManager {
private livekitRoomFactory: () => LivekitRoom; private livekitRoomFactory: () => LivekitRoom;
public constructor( public constructor(
private client: MatrixClient,
private scope: ObservableScope, private scope: ObservableScope,
private client: MatrixClient,
private devices: MediaDevices, private devices: MediaDevices,
private processorState: ProcessorState, private processorState$: Behavior<ProcessorState>,
private e2eeLivekitOptions$: Behavior<E2EEOptions | undefined>, private e2eeLivekitOptions: E2EEOptions | undefined,
private logger?: Logger, private logger?: Logger,
livekitRoomFactory?: () => LivekitRoom, livekitRoomFactory?: () => LivekitRoom,
) { ) {
@@ -55,8 +55,8 @@ export class ConnectionManager {
new LivekitRoom( new LivekitRoom(
generateRoomOption( generateRoomOption(
this.devices, this.devices,
this.processorState, this.processorState$.value,
this.e2eeLivekitOptions$.value, this.e2eeLivekitOptions,
), ),
); );
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;

10
src/state/remoteMembers/displayname.ts
View File

@@ -5,11 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { type Room, type RoomMember, RoomStateEvent } from "matrix-js-sdk"; import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
import { combineLatest, fromEvent, type Observable, startWith } from "rxjs"; import { combineLatest, fromEvent, type Observable, startWith } from "rxjs";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger"; import { logger } from "matrix-js-sdk/lib/logger";
import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix"; import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
// eslint-disable-next-line rxjs/no-internal
import { type HasEventTargetAddRemove } from "rxjs/internal/observable/fromEvent";
import { type ObservableScope } from "../ObservableScope"; import { type ObservableScope } from "../ObservableScope";
import { import {
@@ -22,11 +24,13 @@ import { type Behavior } from "../Behavior";
* Displayname for each member of the call. This will disambiguate * Displayname for each member of the call. This will disambiguate
* any displayname that clashes with another member. Only members * any displayname that clashes with another member. Only members
* joined to the call are considered here. * joined to the call are considered here.
*
* @returns Map<member.id, displayname> uses the rtc member idenitfier as the key.
*/ */
// don't do this work more times than we need to. This is achieved by converting to a behavior: // don't do this work more times than we need to. This is achieved by converting to a behavior:
export const memberDisplaynames$ = ( export const memberDisplaynames$ = (
scope: ObservableScope, scope: ObservableScope,
matrixRoom: Room, matrixRoom: Pick<MatrixRoom, "getMember"> & HasEventTargetAddRemove<unknown>,
memberships$: Observable<CallMembership[]>, memberships$: Observable<CallMembership[]>,
userId: string, userId: string,
deviceId: string, deviceId: string,
@@ -73,7 +77,7 @@ export const memberDisplaynames$ = (
export function getRoomMemberFromRtcMember( export function getRoomMemberFromRtcMember(
rtcMember: CallMembership, rtcMember: CallMembership,
room: MatrixRoom, room: Pick<MatrixRoom, "getMember">,
): { id: string; member: RoomMember | undefined } { ): { id: string; member: RoomMember | undefined } {
return { return {
id: rtcMember.userId + ":" + rtcMember.deviceId, id: rtcMember.userId + ":" + rtcMember.deviceId,

95
src/state/remoteMembers/matrixLivekitMerger.ts
View File

@@ -5,38 +5,23 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { import { type Participant as LivekitParticipant } from "livekit-client";
type RemoteParticipant,
type Participant as LivekitParticipant,
} from "livekit-client";
import { import {
isLivekitTransport, isLivekitTransport,
type LivekitTransport, type LivekitTransport,
type CallMembership, type CallMembership,
} from "matrix-js-sdk/lib/matrixrtc"; } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, startWith, type Observable } from "rxjs"; import { combineLatest, map, startWith, type Observable } from "rxjs";
// eslint-disable-next-line rxjs/no-internal
import { type HasEventTargetAddRemove } from "rxjs/internal/observable/fromEvent";
import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk"; import type { Room as MatrixRoom, RoomMember } from "matrix-js-sdk";
// import type { Logger } from "matrix-js-sdk/lib/logger"; // import type { Logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior"; import { type Behavior } from "../Behavior";
import { type ObservableScope } from "../ObservableScope"; import { type ObservableScope } from "../ObservableScope";
import { type ConnectionManager } from "./ConnectionManager"; import { type ConnectionManager } from "./ConnectionManager";
import { getRoomMemberFromRtcMember } from "./displayname"; import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname";
import { type Connection } from "./Connection";
/**
* Represents participant publishing or expected to publish on the connection.
* It is paired with its associated rtc membership.
*/
export type PublishingParticipant = {
/**
* The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room.
*/
participant: RemoteParticipant | undefined;
/**
* The rtc call membership associated with this participant.
*/
membership: CallMembership;
};
/** /**
* Represent a matrix call member and his associated livekit participation. * Represent a matrix call member and his associated livekit participation.
@@ -45,10 +30,16 @@ export type PublishingParticipant = {
*/ */
export interface MatrixLivekitItem { export interface MatrixLivekitItem {
membership: CallMembership; membership: CallMembership;
livekitParticipant?: LivekitParticipant; displayName: string;
//TODO Try to remove this! Its waaay to much information participant?: LivekitParticipant;
// Just use to get the member's avatar connection?: Connection;
/**
* TODO Try to remove this! Its waaay to much information.
* Just get the member's avatar
* @deprecated
*/
member?: RoomMember; member?: RoomMember;
mxcAvatarUrl?: string;
} }
// Alternative structure idea: // Alternative structure idea:
@@ -73,13 +64,17 @@ export class MatrixLivekitMerger {
// private readonly logger: Logger; // private readonly logger: Logger;
public constructor( public constructor(
private scope: ObservableScope,
private memberships$: Observable<CallMembership[]>, private memberships$: Observable<CallMembership[]>,
private connectionManager: ConnectionManager, private connectionManager: ConnectionManager,
private scope: ObservableScope,
// TODO this is too much information for that class, // TODO this is too much information for that class,
// apparently needed to get a room member to later get the Avatar // apparently needed to get a room member to later get the Avatar
// => Extract an AvatarService instead? // => Extract an AvatarService instead?
private matrixRoom: MatrixRoom, // Better with just `getMember`
private matrixRoom: Pick<MatrixRoom, "getMember"> &
HasEventTargetAddRemove<unknown>,
private userId: string,
private deviceId: string,
// parentLogger: Logger, // parentLogger: Logger,
) { ) {
// this.logger = parentLogger.getChild("MatrixLivekitMerger"); // this.logger = parentLogger.getChild("MatrixLivekitMerger");
@@ -93,6 +88,13 @@ export class MatrixLivekitMerger {
/// PRIVATES /// PRIVATES
// ======================================= // =======================================
private start$(): Observable<MatrixLivekitItem[]> { private start$(): Observable<MatrixLivekitItem[]> {
const displaynameMap$ = memberDisplaynames$(
this.scope,
this.matrixRoom,
this.memberships$,
this.userId,
this.deviceId,
);
const membershipsWithTransport$ = const membershipsWithTransport$ =
this.mapMembershipsToMembershipWithTransport$(); this.mapMembershipsToMembershipWithTransport$();
@@ -101,26 +103,33 @@ export class MatrixLivekitMerger {
return combineLatest([ return combineLatest([
membershipsWithTransport$, membershipsWithTransport$,
this.connectionManager.allParticipantsByMemberId$, this.connectionManager.allParticipantsByMemberId$,
displaynameMap$,
]).pipe( ]).pipe(
map(([memberships, participantsByMemberId]) => { map(([memberships, participantsByMemberId, displayNameMap]) => {
const items = memberships.map(({ membership, transport }) => { const items: MatrixLivekitItem[] = memberships.map(
const participantsWithConnection = participantsByMemberId.get( ({ membership, transport }) => {
membership.membershipID, const participantsWithConnection = participantsByMemberId.get(
); membership.membershipID,
const participant =
transport &&
participantsWithConnection?.find((p) =>
areLivekitTransportsEqual(p.connection.transport, transport),
); );
return { const participant =
livekitParticipant: participant, transport &&
membership, participantsWithConnection?.find((p) =>
// This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar) areLivekitTransportsEqual(p.connection.transport, transport),
member: );
// Why a member error? if we have a call membership there is a room member const member = getRoomMemberFromRtcMember(
getRoomMemberFromRtcMember(membership, this.matrixRoom)?.member, membership,
} as MatrixLivekitItem; this.matrixRoom,
}); )?.member;
return {
...participant,
membership,
// This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar)
member,
displayName: displayNameMap.get(membership.membershipID) ?? "---",
mxcAvatarUrl: member?.getMxcAvatarUrl(),
};
},
);
return items; return items;
}), }),
); );

2
src/utils/displayname.ts
View File

@@ -41,7 +41,7 @@ function removeHiddenChars(str: string): string {
export function shouldDisambiguate( export function shouldDisambiguate(
member: { rawDisplayName?: string; userId: string }, member: { rawDisplayName?: string; userId: string },
memberships: CallMembership[], memberships: CallMembership[],
room: Room, room: Pick<Room, "getMember">,
): boolean { ): boolean {
const { rawDisplayName: displayName, userId } = member; const { rawDisplayName: displayName, userId } = member;
if (!displayName || displayName === userId) return false; if (!displayName || displayName === userId) return false;