studio/element-call
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mvp

Browse Source
This commit is contained in:
Timo K
2025-12-01 12:43:17 +01:00
parent 2d8ffc0ccd
commit 284a52c23c
12 changed files with 296 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
godot/index.html
View File

@@ -20,7 +20,7 @@
await window.matrixRTCSdk.join(); await window.matrixRTCSdk.join();
console.info("matrixRTCSdk joined "); console.info("matrixRTCSdk joined ");
// sdk.data$.subscribe((data) => { // window.matrixRTCSdk.data$.subscribe((data) => {
// console.log(data); // console.log(data);
// const div = document.getElementById("data"); // const div = document.getElementById("data");
// div.appendChild(document.createTextNode(data)); // div.appendChild(document.createTextNode(data));
@@ -36,9 +36,9 @@
</head> </head>
<body> <body>
<button onclick="window.matrixRTCSdk.leave();">Leave</button> <button onclick="window.matrixRTCSdk.leave();">Leave</button>
<!--<button onclick="window.matrixRTCSdk.sendData({prop: 'Hello, world!'});"> <button onclick="window.matrixRTCSdk.sendData({prop: 'Hello, world!'});">
Send Text Send Text
</button>--> </button>
<div id="data"></div> <div id="data"></div>
<canvas id="canvas"></canvas> <canvas id="canvas"></canvas>
</body> </body>

198
godot/main.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/ */
// import { type InitResult } from "../src/ClientContext"; // import { type InitResult } from "../src/ClientContext";
import { map, type Observable, of, Subject, switchMap } from "rxjs"; import { map, type Observable, of, Subject, switchMap, tap } from "rxjs";
import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc"; import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc";
import { type TextStreamInfo } from "livekit-client/dist/src/room/types"; import { type TextStreamInfo } from "livekit-client/dist/src/room/types";
import { import {
@@ -36,7 +36,7 @@ interface MatrixRTCSdk {
/** @throws on leave errors */ /** @throws on leave errors */
leave: () => void; leave: () => void;
data$: Observable<{ sender: string; data: string }>; data$: Observable<{ sender: string; data: string }>;
sendData?: (data: Record<string, unknown>) => Promise<void>; sendData?: (data: unknown) => Promise<void>;
} }
export async function createMatrixRTCSdk(): Promise<MatrixRTCSdk> { export async function createMatrixRTCSdk(): Promise<MatrixRTCSdk> {
logger.info("Hello"); logger.info("Hello");
@@ -67,97 +67,115 @@ export async function createMatrixRTCSdk(): Promise<MatrixRTCSdk> {
// create data listener // create data listener
const data$ = new Subject<{ sender: string; data: string }>(); const data$ = new Subject<{ sender: string; data: string }>();
// const lkTextStreamHandlerFunction = async ( const lkTextStreamHandlerFunction = async (
// reader: TextStreamReader, reader: TextStreamReader,
// participantInfo: { identity: string }, participantInfo: { identity: string },
// livekitRoom: LivekitRoom, livekitRoom: LivekitRoom,
// ): Promise<void> => { ): Promise<void> => {
// const info = reader.info; const info = reader.info;
// console.log( logger.info(
// `Received text stream from ${participantInfo.identity}\n` + `Received text stream from ${participantInfo.identity}\n` +
// ` Topic: ${info.topic}\n` + ` Topic: ${info.topic}\n` +
// ` Timestamp: ${info.timestamp}\n` + ` Timestamp: ${info.timestamp}\n` +
// ` ID: ${info.id}\n` + ` ID: ${info.id}\n` +
// ` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText` ` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
// ); );
// const participants = callViewModel.livekitRoomItems$.value.find( const participants = callViewModel.livekitRoomItems$.value.find(
// (i) => i.livekitRoom === livekitRoom, (i) => i.livekitRoom === livekitRoom,
// )?.participants; )?.participants;
// if (participants && participants.includes(participantInfo.identity)) { if (participants && participants.includes(participantInfo.identity)) {
// const text = await reader.readAll(); const text = await reader.readAll();
// console.log(`Received text: ${text}`); logger.info(`Received text: ${text}`);
// data$.next({ sender: participantInfo.identity, data: text }); data$.next({ sender: participantInfo.identity, data: text });
// } else { } else {
// logger.warn( logger.warn(
// "Received text from unknown participant", "Received text from unknown participant",
// participantInfo.identity, participantInfo.identity,
// ); );
// } }
// }; };
// const livekitRoomItemsSub = callViewModel.livekitRoomItems$ const livekitRoomItemsSub = callViewModel.livekitRoomItems$
// .pipe(currentAndPrev) .pipe(
// .subscribe({ tap((beforecurrentAndPrev) => {
// next: ({ prev, current }) => { logger.info(
// const prevRooms = prev.map((i) => i.livekitRoom); `LiveKit room items updated: ${beforecurrentAndPrev.length}`,
// const currentRooms = current.map((i) => i.livekitRoom); beforecurrentAndPrev,
// const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r)); );
// const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r)); }),
// addedRooms.forEach((r) => currentAndPrev,
// r.registerTextStreamHandler( tap((aftercurrentAndPrev) => {
// TEXT_LK_TOPIC, logger.info(
// (reader, participantInfo) => `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`,
// void lkTextStreamHandlerFunction(reader, participantInfo, r), aftercurrentAndPrev,
// ), );
// ); }),
// removedRooms.forEach((r) => )
// r.unregisterTextStreamHandler(TEXT_LK_TOPIC), .subscribe({
// ); next: ({ prev, current }) => {
// }, const prevRooms = prev.map((i) => i.livekitRoom);
// complete: () => { const currentRooms = current.map((i) => i.livekitRoom);
// logger.info("Livekit room items subscription completed"); const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
// for (const item of callViewModel.livekitRoomItems$.value) { const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
// logger.info("unregistering room item from room", item.url); addedRooms.forEach((r) => {
// item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); logger.info(`Registering text stream handler for room `);
// } r.registerTextStreamHandler(
// }, TEXT_LK_TOPIC,
// }); (reader, participantInfo) =>
void lkTextStreamHandlerFunction(reader, participantInfo, r),
);
});
removedRooms.forEach((r) => {
logger.info(`Unregistering text stream handler for room `);
r.unregisterTextStreamHandler(TEXT_LK_TOPIC);
});
},
complete: () => {
logger.info("Livekit room items subscription completed");
for (const item of callViewModel.livekitRoomItems$.value) {
logger.info("unregistering room item from room", item.url);
item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
}
},
});
// create sendData function // create sendData function
// const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> = const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
// scope.behavior( scope.behavior(
// callViewModel.localMatrixLivekitMember$.pipe( callViewModel.localmatrixLivekitMembers$.pipe(
// switchMap((m) => { switchMap((m) => {
// if (!m) if (!m)
// return of((data: string): never => { return of((data: string): never => {
// throw Error("local membership not yet ready."); throw Error("local membership not yet ready.");
// }); });
// return m.participant$.pipe( return m.participant$.pipe(
// map((p) => { map((p) => {
// if (p === null) { if (p === null) {
// return (data: string): never => { return (data: string): never => {
// throw Error("local participant not yet ready to send data."); throw Error("local participant not yet ready to send data.");
// }; };
// } else { } else {
// return async (data: string): Promise<TextStreamInfo> => return async (data: string): Promise<TextStreamInfo> =>
// p.sendText(data, { topic: TEXT_LK_TOPIC }); p.sendText(data, { topic: TEXT_LK_TOPIC });
// } }
// }), }),
// ); );
// }), }),
// ), ),
// ); );
// const sendData = async (data: Record<string, unknown>): Promise<void> => { const sendData = async (data: unknown): Promise<void> => {
// const dataString = JSON.stringify(data); const dataString = JSON.stringify(data);
// try { logger.info("try sending: ", dataString);
// const info = await sendFn.value(dataString); try {
// logger.info(`Sent text with stream ID: ${info.id}`); await Promise.resolve();
// } catch (e) { const info = await sendFn.value(dataString);
// console.error("failed sending: ", dataString, e); logger.info(`Sent text with stream ID: ${info.id}`);
// } } catch (e) {
// }; logger.error("failed sending: ", dataString, e);
}
};
// after hangup gets called // after hangup gets called
const leaveSubs = callViewModel.leave$.subscribe(() => { const leaveSubs = callViewModel.leave$.subscribe(() => {
@@ -202,9 +220,9 @@ export async function createMatrixRTCSdk(): Promise<MatrixRTCSdk> {
leave: (): void => { leave: (): void => {
callViewModel.hangup(); callViewModel.hangup();
leaveSubs.unsubscribe(); leaveSubs.unsubscribe();
// livekitRoomItemsSub.unsubscribe(); livekitRoomItemsSub.unsubscribe();
}, },
data$, data$,
// sendData, sendData,
}; };
} }

24
src/state/CallViewModel/CallViewModel.ts
View File

@@ -264,7 +264,7 @@ export interface CallViewModel {
livekitRoomItems$: Behavior<LivekitRoomItem[]>; livekitRoomItems$: Behavior<LivekitRoomItem[]>;
/** use the layout instead, this is just for the godot export. */ /** use the layout instead, this is just for the godot export. */
userMedia$: Behavior<UserMedia[]>; userMedia$: Behavior<UserMedia[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>; localmatrixLivekitMembers$: Behavior<LocalMatrixLivekitMember | null>;
/** List of participants raising their hand */ /** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>; handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -449,7 +449,7 @@ export function createCallViewModel$(
logger: logger, logger: logger,
}); });
const matrixLivekitMembers$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: scope, scope: scope,
membershipsWithTransport$: membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$, membershipsAndTransports.membershipsWithTransport$,
@@ -515,7 +515,7 @@ export function createCallViewModel$(
userId: userId, userId: userId,
}; };
const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> = const localmatrixLivekitMembers$: Behavior<LocalMatrixLivekitMember | null> =
scope.behavior( scope.behavior(
localRtcMembership$.pipe( localRtcMembership$.pipe(
switchMap((membership) => { switchMap((membership) => {
@@ -607,8 +607,11 @@ export function createCallViewModel$(
const reconnecting$ = localMembership.reconnecting$; const reconnecting$ = localMembership.reconnecting$;
const pretendToBeDisconnected$ = reconnecting$; const pretendToBeDisconnected$ = reconnecting$;
const audioParticipants$ = scope.behavior( const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe( matrixLivekitMembers$.pipe(
tap((val) => {
logger.debug("matrixLivekitMembers$ updated", val.value);
}),
switchMap((membersWithEpoch) => { switchMap((membersWithEpoch) => {
const members = membersWithEpoch.value; const members = membersWithEpoch.value;
const a$ = combineLatest( const a$ = combineLatest(
@@ -649,6 +652,12 @@ export function createCallViewModel$(
return acc; return acc;
}, []), }, []),
), ),
tap((val) => {
logger.debug(
"livekitRoomItems$ updated",
val.map((v) => v.url),
);
}),
), ),
[], [],
); );
@@ -676,7 +685,7 @@ export function createCallViewModel$(
*/ */
const userMedia$ = scope.behavior<UserMedia[]>( const userMedia$ = scope.behavior<UserMedia[]>(
combineLatest([ combineLatest([
localMatrixLivekitMember$, localmatrixLivekitMembers$,
matrixLivekitMembers$, matrixLivekitMembers$,
duplicateTiles.value$, duplicateTiles.value$,
]).pipe( ]).pipe(
@@ -1489,8 +1498,7 @@ export function createCallViewModel$(
), ),
participantCount$: participantCount$, participantCount$: participantCount$,
livekitRoomItems$: audioParticipants$, livekitRoomItems$,
handsRaised$: handsRaised$, handsRaised$: handsRaised$,
reactions$: reactions$, reactions$: reactions$,
joinSoundEffect$: joinSoundEffect$, joinSoundEffect$: joinSoundEffect$,
@@ -1510,7 +1518,7 @@ export function createCallViewModel$(
pip$: pip$, pip$: pip$,
layout$: layout$, layout$: layout$,
userMedia$, userMedia$,
localMatrixLivekitMember$, localmatrixLivekitMembers$,
tileStoreGeneration$: tileStoreGeneration$, tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$, showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$, showSpeakingIndicators$: showSpeakingIndicators$,

33
src/state/CallViewModel/localMember/Publisher.ts
View File

@@ -56,15 +56,15 @@ export class Publisher {
devices: MediaDevices, devices: MediaDevices,
private readonly muteStates: MuteStates, private readonly muteStates: MuteStates,
trackerProcessorState$: Behavior<ProcessorState>, trackerProcessorState$: Behavior<ProcessorState>,
private logger?: Logger, private logger: Logger,
) { ) {
this.logger?.info("[PublishConnection] Create LiveKit room"); this.logger.info("[PublishConnection] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams(); const { controlledAudioDevices } = getUrlParams();
const room = connection.livekitRoom; const room = connection.livekitRoom;
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => { room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
this.logger?.error("Failed to set E2EE enabled on room", e); this.logger.error("Failed to set E2EE enabled on room", e);
}); });
// Setup track processor syncing (blur) // Setup track processor syncing (blur)
@@ -74,7 +74,7 @@ export class Publisher {
this.workaroundRestartAudioInputTrackChrome(devices, scope); this.workaroundRestartAudioInputTrackChrome(devices, scope);
this.scope.onEnd(() => { this.scope.onEnd(() => {
this.logger?.info( this.logger.info(
"[PublishConnection] Scope ended -> stop publishing all tracks", "[PublishConnection] Scope ended -> stop publishing all tracks",
); );
void this.stopPublishing(); void this.stopPublishing();
@@ -132,13 +132,14 @@ export class Publisher {
video, video,
}) })
.catch((error) => { .catch((error) => {
this.logger?.error("Failed to create tracks", error); this.logger.error("Failed to create tracks", error);
})) ?? []; })) ?? [];
} }
return this.tracks; return this.tracks;
} }
public async startPublishing(): Promise<LocalTrack[]> { public async startPublishing(): Promise<LocalTrack[]> {
this.logger.info("Start publishing");
const lkRoom = this.connection.livekitRoom; const lkRoom = this.connection.livekitRoom;
const { promise, resolve, reject } = Promise.withResolvers<void>(); const { promise, resolve, reject } = Promise.withResolvers<void>();
const sub = this.connection.state$.subscribe((s) => { const sub = this.connection.state$.subscribe((s) => {
@@ -150,7 +151,7 @@ export class Publisher {
reject(new Error("Failed to connect to LiveKit server")); reject(new Error("Failed to connect to LiveKit server"));
break; break;
default: default:
this.logger?.info("waiting for connection: ", s.state); this.logger.info("waiting for connection: ", s.state);
} }
}); });
try { try {
@@ -160,12 +161,14 @@ export class Publisher {
} finally { } finally {
sub.unsubscribe(); sub.unsubscribe();
} }
this.logger.info("publish ", this.tracks.length, "tracks");
for (const track of this.tracks) { for (const track of this.tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout. // with a timeout.
await lkRoom.localParticipant.publishTrack(track).catch((error) => { await lkRoom.localParticipant.publishTrack(track).catch((error) => {
this.logger?.error("Failed to publish track", error); this.logger.error("Failed to publish track", error);
}); });
this.logger.info("published track ", track.kind, track.id);
// TODO: check if the connection is still active? and break the loop if not? // TODO: check if the connection is still active? and break the loop if not?
} }
@@ -229,7 +232,7 @@ export class Publisher {
.getTrackPublication(Track.Source.Microphone) .getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack() ?.audioTrack?.restartTrack()
.catch((e) => { .catch((e) => {
this.logger?.error(`Failed to restart audio device track`, e); this.logger.error(`Failed to restart audio device track`, e);
}); });
} }
}); });
@@ -249,7 +252,7 @@ export class Publisher {
selected$.pipe(scope.bind()).subscribe((device) => { selected$.pipe(scope.bind()).subscribe((device) => {
if (lkRoom.state != LivekitConnectionState.Connected) return; if (lkRoom.state != LivekitConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return; // if (this.connectionState$.value !== ConnectionState.Connected) return;
this.logger?.info( this.logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
lkRoom.getActiveDevice(kind), lkRoom.getActiveDevice(kind),
" !== ", " !== ",
@@ -262,7 +265,7 @@ export class Publisher {
lkRoom lkRoom
.switchActiveDevice(kind, device.id) .switchActiveDevice(kind, device.id)
.catch((e: Error) => .catch((e: Error) =>
this.logger?.error( this.logger.error(
`Failed to sync ${kind} device with LiveKit`, `Failed to sync ${kind} device with LiveKit`,
e, e,
), ),
@@ -287,10 +290,7 @@ export class Publisher {
try { try {
await lkRoom.localParticipant.setMicrophoneEnabled(desired); await lkRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) { } catch (e) {
this.logger?.error( this.logger.error("Failed to update LiveKit audio input mute state", e);
"Failed to update LiveKit audio input mute state",
e,
);
} }
return lkRoom.localParticipant.isMicrophoneEnabled; return lkRoom.localParticipant.isMicrophoneEnabled;
}); });
@@ -298,10 +298,7 @@ export class Publisher {
try { try {
await lkRoom.localParticipant.setCameraEnabled(desired); await lkRoom.localParticipant.setCameraEnabled(desired);
} catch (e) { } catch (e) {
this.logger?.error( this.logger.error("Failed to update LiveKit video input mute state", e);
"Failed to update LiveKit video input mute state",
e,
);
} }
return lkRoom.localParticipant.isCameraEnabled; return lkRoom.localParticipant.isCameraEnabled;
}); });

28
src/state/CallViewModel/remoteMembers/Connection.test.ts
View File

@@ -382,17 +382,15 @@ describe("Publishing participants observations", () => {
const bobIsAPublisher = Promise.withResolvers<void>(); const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>(); const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: PublishingParticipant[][] = []; const observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipantsWithTracks$.subscribe( const s = connection.remoteParticipants$.subscribe((publishers) => {
(publishers) => { observedPublishers.push(publishers);
observedPublishers.push(publishers); if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) {
if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { bobIsAPublisher.resolve();
bobIsAPublisher.resolve(); }
} if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) {
if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { danIsAPublisher.resolve();
danIsAPublisher.resolve(); }
} });
},
);
onTestFinished(() => s.unsubscribe()); onTestFinished(() => s.unsubscribe());
// The publishingParticipants$ observable is derived from the current members of the // The publishingParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing // livekitRoom and the rtc membership in order to publish the members that are publishing
@@ -437,11 +435,9 @@ describe("Publishing participants observations", () => {
const connection = setupRemoteConnection(); const connection = setupRemoteConnection();
let observedPublishers: PublishingParticipant[][] = []; let observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipantsWithTracks$.subscribe( const s = connection.remoteParticipants$.subscribe((publishers) => {
(publishers) => { observedPublishers.push(publishers);
observedPublishers.push(publishers); });
},
);
onTestFinished(() => s.unsubscribe()); onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [ let participants: RemoteParticipant[] = [

29
src/state/CallViewModel/remoteMembers/Connection.ts
View File

@@ -19,7 +19,7 @@ import {
RoomEvent, RoomEvent,
} from "livekit-client"; } from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map, type Observable } from "rxjs"; import { BehaviorSubject, type Observable } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger"; import { type Logger } from "matrix-js-sdk/lib/logger";
import { import {
@@ -146,6 +146,10 @@ export class Connection {
transport: this.transport, transport: this.transport,
livekitConnectionState$: connectionStateObserver(this.livekitRoom), livekitConnectionState$: connectionStateObserver(this.livekitRoom),
}); });
this.logger.info(
"Connected to LiveKit room",
this.transport.livekit_service_url,
);
} catch (error) { } catch (error) {
this.logger.debug(`Failed to connect to LiveKit room: ${error}`); this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
this._state$.next({ this._state$.next({
@@ -189,9 +193,7 @@ export class Connection {
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection. * It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/ */
public readonly remoteParticipantsWithTracks$: Behavior< public readonly remoteParticipants$: Behavior<PublishingParticipant[]>;
PublishingParticipant[]
>;
/** /**
* The media transport to connect to. * The media transport to connect to.
@@ -213,7 +215,7 @@ export class Connection {
public constructor(opts: ConnectionOpts, logger: Logger) { public constructor(opts: ConnectionOpts, logger: Logger) {
this.logger = logger.getChild("[Connection]"); this.logger = logger.getChild("[Connection]");
this.logger.info( this.logger.info(
`[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
); );
const { transport, client, scope } = opts; const { transport, client, scope } = opts;
@@ -223,20 +225,21 @@ export class Connection {
// REMOTE participants with track!!! // REMOTE participants with track!!!
// this.remoteParticipantsWithTracks$ // this.remoteParticipantsWithTracks$
this.remoteParticipantsWithTracks$ = scope.behavior( this.remoteParticipants$ = scope.behavior(
// only tracks remote participants // only tracks remote participants
connectedParticipantsObserver(this.livekitRoom, { connectedParticipantsObserver(this.livekitRoom, {
additionalRoomEvents: [ additionalRoomEvents: [
RoomEvent.TrackPublished, RoomEvent.TrackPublished,
RoomEvent.TrackUnpublished, RoomEvent.TrackUnpublished,
], ],
}).pipe( }),
map((participants) => { // .pipe(
return participants.filter( // map((participants) => {
(participant) => participant.getTrackPublications().length > 0, // return participants.filter(
); // (participant) => participant.getTrackPublications().length > 0,
}), // );
), // }),
// )
[], [],
); );

2
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
View File

@@ -13,7 +13,7 @@ import {
type BaseKeyProvider, type BaseKeyProvider,
} from "livekit-client"; } from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger"; import { type Logger } from "matrix-js-sdk/lib/logger";
import E2EEWorker from "livekit-client/e2ee-worker?worker"; import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type ObservableScope } from "../../ObservableScope.ts"; import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts"; import { Connection } from "./Connection.ts";

52
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
View File

@@ -289,47 +289,47 @@ describe("connectionManagerData$ stream", () => {
a: expect.toSatisfy((e) => { a: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value; const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2); expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
return true; return true;
}), }),
b: expect.toSatisfy((e) => { b: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value; const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2); expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( expect(
"user1A", data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
); ).toBe("user1A");
return true; return true;
}), }),
c: expect.toSatisfy((e) => { c: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value; const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2); expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( expect(
"user1A", data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
); ).toBe("user1A");
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( expect(
"user2A", data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
); ).toBe("user2A");
return true; return true;
}), }),
d: expect.toSatisfy((e) => { d: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value; const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2); expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( expect(
"user1A", data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
); ).toBe("user1A");
expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe( expect(
"user1B", data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
); ).toBe("user1B");
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( expect(
"user2A", data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
); ).toBe("user2A");
return true; return true;
}), }),
}); });

39
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -24,7 +24,10 @@ import { type ConnectionFactory } from "./ConnectionFactory.ts";
export class ConnectionManagerData { export class ConnectionManagerData {
private readonly store: Map< private readonly store: Map<
string, string,
[Connection, (LocalParticipant | RemoteParticipant)[]] {
connection: Connection;
participants: (LocalParticipant | RemoteParticipant)[];
}
> = new Map(); > = new Map();
public constructor() {} public constructor() {}
@@ -36,9 +39,9 @@ export class ConnectionManagerData {
const key = this.getKey(connection.transport); const key = this.getKey(connection.transport);
const existing = this.store.get(key); const existing = this.store.get(key);
if (!existing) { if (!existing) {
this.store.set(key, [connection, participants]); this.store.set(key, { connection, participants });
} else { } else {
existing[1].push(...participants); existing.participants.push(...participants);
} }
} }
@@ -47,25 +50,26 @@ export class ConnectionManagerData {
} }
public getConnections(): Connection[] { public getConnections(): Connection[] {
return Array.from(this.store.values()).map(([connection]) => connection); return Array.from(this.store.values()).map(({ connection }) => connection);
} }
public getConnectionForTransport( public getConnectionForTransport(
transport: LivekitTransport, transport: LivekitTransport,
): Connection | null { ): Connection | null {
return this.store.get(this.getKey(transport))?.[0] ?? null; return this.store.get(this.getKey(transport))?.connection ?? null;
} }
public getParticipantForTransport( public getParticipantsForTransport(
transport: LivekitTransport, transport: LivekitTransport,
): (LocalParticipant | RemoteParticipant)[] { ): (LocalParticipant | RemoteParticipant)[] {
const key = transport.livekit_service_url + "|" + transport.livekit_alias; const key = transport.livekit_service_url + "|" + transport.livekit_alias;
const existing = this.store.get(key); const existing = this.store.get(key);
if (existing) { if (existing) {
return existing[1]; return existing.participants;
} }
return []; return [];
} }
/** /**
* Get all connections where the given participant is publishing. * Get all connections where the given participant is publishing.
* In theory, there could be several connections where the same participant is publishing but with * In theory, there could be several connections where the same participant is publishing but with
@@ -76,8 +80,12 @@ export class ConnectionManagerData {
participantId: ParticipantId, participantId: ParticipantId,
): Connection[] { ): Connection[] {
const connections: Connection[] = []; const connections: Connection[] = [];
for (const [connection, participants] of this.store.values()) { for (const { connection, participants } of this.store.values()) {
if (participants.some((p) => p.identity === participantId)) { if (
participants.some(
(participant) => participant?.identity === participantId,
)
) {
connections.push(connection); connections.push(connection);
} }
} }
@@ -183,23 +191,24 @@ export function createConnectionManager$({
const epoch = connections.epoch; const epoch = connections.epoch;
// Map the connections to list of {connection, participants}[] // Map the connections to list of {connection, participants}[]
const listOfConnectionsWithPublishingParticipants = const listOfConnectionsWithParticipants = connections.value.map(
connections.value.map((connection) => { (connection) => {
return connection.remoteParticipantsWithTracks$.pipe( return connection.remoteParticipants$.pipe(
map((participants) => ({ map((participants) => ({
connection, connection,
participants, participants,
})), })),
); );
}); },
);
// probably not required // probably not required
if (listOfConnectionsWithPublishingParticipants.length === 0) { if (listOfConnectionsWithParticipants.length === 0) {
return of(new Epoch(new ConnectionManagerData(), epoch)); return of(new Epoch(new ConnectionManagerData(), epoch));
} }
// combineLatest the several streams into a single stream with the ConnectionManagerData // combineLatest the several streams into a single stream with the ConnectionManagerData
return combineLatest(listOfConnectionsWithPublishingParticipants).pipe( return combineLatest(listOfConnectionsWithParticipants).pipe(
map( map(
(lists) => (lists) =>
new Epoch( new Epoch(

121
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => {
}), }),
); );
const matrixLivekitMember$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: testScope, scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: { connectionManager: {
@@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => {
} as unknown as IConnectionManager, } as unknown as IConnectionManager,
}); });
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
a: expect.toSatisfy((data: MatrixLivekitMember[]) => { "a",
expect(data.length).toEqual(1); {
expectObservable(data[0].membership$).toBe("a", { a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: bobMembership, expect(data.length).toEqual(1);
}); expectObservable(data[0].membership$).toBe("a", {
expectObservable(data[0].participant$).toBe("a", { a: bobMembership,
a: null, });
}); expectObservable(data[0].participant$).toBe("a", {
expectObservable(data[0].connection$).toBe("a", { a: null,
a: null, });
}); expectObservable(data[0].connection$).toBe("a", {
return true; a: null,
}), });
}); return true;
}),
},
);
}); });
}); });
@@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => {
}), }),
); );
const matrixLivekitMember$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: testScope, scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: { connectionManager: {
@@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => {
} as unknown as IConnectionManager, } as unknown as IConnectionManager,
}); });
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
a: expect.toSatisfy((data: MatrixLivekitMember[]) => { "a",
expect(data.length).toEqual(1); {
expectObservable(data[0].membership$).toBe("a", { a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: bobMembership, expect(data.length).toEqual(1);
}); expectObservable(data[0].membership$).toBe("a", {
expectObservable(data[0].participant$).toBe("a", { a: bobMembership,
a: expect.toSatisfy((participant) => { });
expect(participant).toBeDefined(); expectObservable(data[0].participant$).toBe("a", {
expect(participant!.identity).toEqual(bobParticipantId); a: expect.toSatisfy((participant) => {
return true; expect(participant).toBeDefined();
}), expect(participant!.identity).toEqual(bobParticipantId);
}); return true;
expectObservable(data[0].connection$).toBe("a", { }),
a: connection, });
}); expectObservable(data[0].connection$).toBe("a", {
return true; a: connection,
}), });
}); return true;
}),
},
);
}); });
}); });
@@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => {
}), }),
); );
const matrixLivekitMember$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: testScope, scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: { connectionManager: {
@@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => {
} as unknown as IConnectionManager, } as unknown as IConnectionManager,
}); });
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
a: expect.toSatisfy((data: MatrixLivekitMember[]) => { "a",
expect(data.length).toEqual(1); {
expectObservable(data[0].membership$).toBe("a", { a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: bobMembership, expect(data.length).toEqual(1);
}); expectObservable(data[0].membership$).toBe("a", {
expectObservable(data[0].participant$).toBe("a", { a: bobMembership,
a: null, });
}); expectObservable(data[0].participant$).toBe("a", {
expectObservable(data[0].connection$).toBe("a", { a: null,
a: connection, });
}); expectObservable(data[0].connection$).toBe("a", {
return true; a: connection,
}), });
}); return true;
}),
},
);
}); });
}); });
@@ -283,7 +292,7 @@ describe("Publication edge case", () => {
}), }),
); );
const matrixLivekitMember$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: testScope, scope: testScope,
membershipsWithTransport$: testScope.behavior( membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$, membershipsWithTransport$,
@@ -293,7 +302,7 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager, } as unknown as IConnectionManager,
}); });
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a", "a",
{ {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => { a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
@@ -349,7 +358,7 @@ describe("Publication edge case", () => {
}), }),
); );
const matrixLivekitMember$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: testScope, scope: testScope,
membershipsWithTransport$: testScope.behavior( membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$, membershipsWithTransport$,
@@ -359,7 +368,7 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager, } as unknown as IConnectionManager,
}); });
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a", "a",
{ {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => { a: expect.toSatisfy((data: MatrixLivekitMember[]) => {

11
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -61,12 +61,12 @@ export function createMatrixLivekitMembers$({
scope, scope,
membershipsWithTransport$, membershipsWithTransport$,
connectionManager, connectionManager,
}: Props): Behavior<Epoch<MatrixLivekitMember[]>> { }: Props): { matrixLivekitMembers$: Behavior<Epoch<MatrixLivekitMember[]>> } {
/** /**
* Stream of all the call members and their associated livekit data (if available). * Stream of all the call members and their associated livekit data (if available).
*/ */
return scope.behavior( const matrixLivekitMembers$ = scope.behavior(
combineLatest([ combineLatest([
membershipsWithTransport$, membershipsWithTransport$,
connectionManager.connectionManagerData$, connectionManager.connectionManagerData$,
@@ -91,7 +91,7 @@ export function createMatrixLivekitMembers$({
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
const participants = transport const participants = transport
? managerData.getParticipantForTransport(transport) ? managerData.getParticipantsForTransport(transport)
: []; : [];
const participant = const participant =
participants.find((p) => p.identity == participantId) ?? null; participants.find((p) => p.identity == participantId) ?? null;
@@ -121,6 +121,11 @@ export function createMatrixLivekitMembers$({
), ),
), ),
); );
return {
matrixLivekitMembers$,
// TODO add only publishing participants... maybe. disucss at least
// scope.behavior(matrixLivekitMembers$.pipe(map((items) => items.value.map((i)=>{ i.}))))
};
} }
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)

4
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -124,14 +124,14 @@ test("bob, carl, then bob joining no tracks yet", () => {
logger: logger, logger: logger,
}); });
const matrixLivekitItems$ = createMatrixLivekitMembers$({ const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
scope: testScope, scope: testScope,
membershipsWithTransport$: membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$, membershipsAndTransports.membershipsWithTransport$,
connectionManager, connectionManager,
}); });
expectObservable(matrixLivekitItems$).toBe(vMarble, { expectObservable(matrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => { a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
const items = e.value; const items = e.value;
expect(items.length).toBe(1); expect(items.length).toBe(1);